2024-12-06 05:26:21,094 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-12-06 05:26:21,105 main DEBUG Took 0.009285 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-12-06 05:26:21,106 main DEBUG PluginManager 'Core' found 129 plugins
2024-12-06 05:26:21,106 main DEBUG PluginManager 'Level' found 0 plugins
2024-12-06 05:26:21,107 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-12-06 05:26:21,108 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 05:26:21,114 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-12-06 05:26:21,125 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 05:26:21,126 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 05:26:21,127 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 05:26:21,127 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 05:26:21,128 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 05:26:21,128 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 05:26:21,129 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 05:26:21,129 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 05:26:21,130 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 05:26:21,130 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 05:26:21,131 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 05:26:21,131 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 05:26:21,132 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 05:26:21,132 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 05:26:21,132 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 05:26:21,133 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 05:26:21,133 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 05:26:21,133 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 05:26:21,134 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 05:26:21,134 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 05:26:21,134 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 05:26:21,135 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 05:26:21,135 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 05:26:21,135 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 05:26:21,136 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 05:26:21,136 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-12-06 05:26:21,137 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 05:26:21,138 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-12-06 05:26:21,140 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-12-06 05:26:21,141 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-12-06 05:26:21,142 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-12-06 05:26:21,142 main DEBUG PluginManager 'Converter' found 47 plugins
2024-12-06 05:26:21,150 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-12-06 05:26:21,152 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-12-06 05:26:21,154 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-12-06 05:26:21,154 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-12-06 05:26:21,154 main DEBUG createAppenders(={Console})
2024-12-06 05:26:21,155 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-12-06 05:26:21,155 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-12-06 05:26:21,156 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-12-06 05:26:21,156 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-12-06 05:26:21,156 main DEBUG OutputStream closed
2024-12-06 05:26:21,157 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-12-06 05:26:21,157 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-12-06 05:26:21,157 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-12-06 05:26:21,221 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-12-06 05:26:21,223 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-12-06 05:26:21,225 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-12-06 05:26:21,226 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-12-06 05:26:21,226 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-12-06 05:26:21,227 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-12-06 05:26:21,227 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-12-06 05:26:21,227 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-12-06 05:26:21,228 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-12-06 05:26:21,228 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-12-06 05:26:21,228 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-12-06 05:26:21,228 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-12-06 05:26:21,229 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-12-06 05:26:21,229 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-12-06 05:26:21,229 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-12-06 05:26:21,230 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-12-06 05:26:21,230 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-12-06 05:26:21,230 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-12-06 05:26:21,232 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-06 05:26:21,233 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-12-06 05:26:21,233 main DEBUG Shutdown hook enabled. Registering a new one.
2024-12-06 05:26:21,234 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-12-06T05:26:21,448 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63
2024-12-06 05:26:21,451 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-12-06 05:26:21,452 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-06T05:26:21,460 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-12-06T05:26:21,493 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=151, ProcessCount=11, AvailableMemoryMB=8403
2024-12-06T05:26:21,497 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-06T05:26:21,514 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/cluster_5e02bba0-7a73-8521-fbdd-f73238a7d072, deleteOnExit=true
2024-12-06T05:26:21,515 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-06T05:26:21,516 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/test.cache.data in system properties and HBase conf
2024-12-06T05:26:21,517 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/hadoop.tmp.dir in system properties and HBase conf
2024-12-06T05:26:21,517 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/hadoop.log.dir in system properties and HBase conf
2024-12-06T05:26:21,518 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-06T05:26:21,519 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-06T05:26:21,519 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-06T05:26:21,609 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-12-06T05:26:21,693 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-06T05:26:21,696 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-06T05:26:21,697 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-06T05:26:21,697 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-06T05:26:21,698 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T05:26:21,698 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-06T05:26:21,698 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-06T05:26:21,699 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T05:26:21,699 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T05:26:21,700 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-06T05:26:21,700 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/nfs.dump.dir in system properties and HBase conf
2024-12-06T05:26:21,700 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/java.io.tmpdir in system properties and HBase conf
2024-12-06T05:26:21,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T05:26:21,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-06T05:26:21,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-06T05:26:22,126 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T05:26:22,816 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-12-06T05:26:22,882 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2492ms to org.eclipse.jetty.util.log.Slf4jLog
2024-12-06T05:26:22,945 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:26:23,004 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:26:23,024 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:26:23,025 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:26:23,026 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T05:26:23,039 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:26:23,041 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:26:23,043 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:26:23,222 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/java.io.tmpdir/jetty-localhost-37731-hadoop-hdfs-3_4_1-tests_jar-_-any-16643415143658701835/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T05:26:23,229 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:37731}
2024-12-06T05:26:23,229 INFO  [Time-limited test {}] server.Server(415): Started @2840ms
2024-12-06T05:26:23,254 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T05:26:23,757 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:26:23,763 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:26:23,764 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:26:23,764 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:26:23,764 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T05:26:23,766 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:26:23,767 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:26:23,860 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/java.io.tmpdir/jetty-localhost-40633-hadoop-hdfs-3_4_1-tests_jar-_-any-14570927641083663702/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:26:23,861 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:40633}
2024-12-06T05:26:23,861 INFO  [Time-limited test {}] server.Server(415): Started @3472ms
2024-12-06T05:26:23,905 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:26:24,006 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:26:24,010 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:26:24,012 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:26:24,012 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:26:24,012 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T05:26:24,013 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:26:24,013 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:26:24,109 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/java.io.tmpdir/jetty-localhost-42311-hadoop-hdfs-3_4_1-tests_jar-_-any-8336855138874322528/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:26:24,110 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:42311}
2024-12-06T05:26:24,111 INFO  [Time-limited test {}] server.Server(415): Started @3722ms
2024-12-06T05:26:24,113 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:26:25,166 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/cluster_5e02bba0-7a73-8521-fbdd-f73238a7d072/data/data1/current/BP-225256256-172.17.0.3-1733462782207/current, will proceed with Du for space computation calculation, 
2024-12-06T05:26:25,166 WARN  [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/cluster_5e02bba0-7a73-8521-fbdd-f73238a7d072/data/data2/current/BP-225256256-172.17.0.3-1733462782207/current, will proceed with Du for space computation calculation, 
2024-12-06T05:26:25,166 WARN  [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/cluster_5e02bba0-7a73-8521-fbdd-f73238a7d072/data/data3/current/BP-225256256-172.17.0.3-1733462782207/current, will proceed with Du for space computation calculation, 
2024-12-06T05:26:25,166 WARN  [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/cluster_5e02bba0-7a73-8521-fbdd-f73238a7d072/data/data4/current/BP-225256256-172.17.0.3-1733462782207/current, will proceed with Du for space computation calculation, 
2024-12-06T05:26:25,199 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:26:25,199 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:26:25,243 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd6d22248db176179 with lease ID 0x15e39785feeb66b5: Processing first storage report for DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e from datanode DatanodeRegistration(127.0.0.1:36161, datanodeUuid=d385fa8c-62ea-4020-9b21-63c6656c2f8e, infoPort=38339, infoSecurePort=0, ipcPort=42083, storageInfo=lv=-57;cid=testClusterID;nsid=403529912;c=1733462782207)
2024-12-06T05:26:25,244 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd6d22248db176179 with lease ID 0x15e39785feeb66b5: from storage DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e node DatanodeRegistration(127.0.0.1:36161, datanodeUuid=d385fa8c-62ea-4020-9b21-63c6656c2f8e, infoPort=38339, infoSecurePort=0, ipcPort=42083, storageInfo=lv=-57;cid=testClusterID;nsid=403529912;c=1733462782207), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0
2024-12-06T05:26:25,244 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xae91fcf711c4d827 with lease ID 0x15e39785feeb66b4: Processing first storage report for DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2 from datanode DatanodeRegistration(127.0.0.1:36807, datanodeUuid=3981a4fa-ccc0-4677-9f17-d0559b473553, infoPort=37683, infoSecurePort=0, ipcPort=44633, storageInfo=lv=-57;cid=testClusterID;nsid=403529912;c=1733462782207)
2024-12-06T05:26:25,245 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xae91fcf711c4d827 with lease ID 0x15e39785feeb66b4: from storage DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2 node DatanodeRegistration(127.0.0.1:36807, datanodeUuid=3981a4fa-ccc0-4677-9f17-d0559b473553, infoPort=37683, infoSecurePort=0, ipcPort=44633, storageInfo=lv=-57;cid=testClusterID;nsid=403529912;c=1733462782207), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:26:25,245 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd6d22248db176179 with lease ID 0x15e39785feeb66b5: Processing first storage report for DS-fd9e1367-3a1f-4e36-afe5-f7d362994d8a from datanode DatanodeRegistration(127.0.0.1:36161, datanodeUuid=d385fa8c-62ea-4020-9b21-63c6656c2f8e, infoPort=38339, infoSecurePort=0, ipcPort=42083, storageInfo=lv=-57;cid=testClusterID;nsid=403529912;c=1733462782207)
2024-12-06T05:26:25,245 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd6d22248db176179 with lease ID 0x15e39785feeb66b5: from storage DS-fd9e1367-3a1f-4e36-afe5-f7d362994d8a node DatanodeRegistration(127.0.0.1:36161, datanodeUuid=d385fa8c-62ea-4020-9b21-63c6656c2f8e, infoPort=38339, infoSecurePort=0, ipcPort=42083, storageInfo=lv=-57;cid=testClusterID;nsid=403529912;c=1733462782207), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:26:25,245 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xae91fcf711c4d827 with lease ID 0x15e39785feeb66b4: Processing first storage report for DS-1203df2e-5087-41a1-93df-184b362d3866 from datanode DatanodeRegistration(127.0.0.1:36807, datanodeUuid=3981a4fa-ccc0-4677-9f17-d0559b473553, infoPort=37683, infoSecurePort=0, ipcPort=44633, storageInfo=lv=-57;cid=testClusterID;nsid=403529912;c=1733462782207)
2024-12-06T05:26:25,246 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xae91fcf711c4d827 with lease ID 0x15e39785feeb66b4: from storage DS-1203df2e-5087-41a1-93df-184b362d3866 node DatanodeRegistration(127.0.0.1:36807, datanodeUuid=3981a4fa-ccc0-4677-9f17-d0559b473553, infoPort=37683, infoSecurePort=0, ipcPort=44633, storageInfo=lv=-57;cid=testClusterID;nsid=403529912;c=1733462782207), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:26:25,333 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63
2024-12-06T05:26:25,394 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/cluster_5e02bba0-7a73-8521-fbdd-f73238a7d072/zookeeper_0, clientPort=49487, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/cluster_5e02bba0-7a73-8521-fbdd-f73238a7d072/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/cluster_5e02bba0-7a73-8521-fbdd-f73238a7d072/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-06T05:26:25,403 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49487
2024-12-06T05:26:25,412 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:26:25,414 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:26:25,613 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741825_1001 (size=7)
2024-12-06T05:26:25,613 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741825_1001 (size=7)
2024-12-06T05:26:26,023 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1 with version=8
2024-12-06T05:26:26,024 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/hbase-staging
2024-12-06T05:26:26,095 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-12-06T05:26:26,327 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/ec46afd49254:0 server-side Connection retries=45
2024-12-06T05:26:26,336 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:26:26,336 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T05:26:26,340 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T05:26:26,340 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:26:26,340 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T05:26:26,455 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-06T05:26:26,509 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-12-06T05:26:26,517 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-12-06T05:26:26,520 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T05:26:26,541 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 70737 (auto-detected)
2024-12-06T05:26:26,541 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected)
2024-12-06T05:26:26,559 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39099
2024-12-06T05:26:26,578 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39099 connecting to ZooKeeper ensemble=127.0.0.1:49487
2024-12-06T05:26:26,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:390990x0, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T05:26:26,705 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39099-0x101a91af90e0000 connected
2024-12-06T05:26:26,816 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:26:26,821 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:26:26,835 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:26:26,839 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1, hbase.cluster.distributed=false
2024-12-06T05:26:26,865 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T05:26:26,869 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39099
2024-12-06T05:26:26,870 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39099
2024-12-06T05:26:26,870 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39099
2024-12-06T05:26:26,871 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39099
2024-12-06T05:26:26,871 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39099
2024-12-06T05:26:26,963 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/ec46afd49254:0 server-side Connection retries=45
2024-12-06T05:26:26,964 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:26:26,964 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T05:26:26,965 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T05:26:26,965 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:26:26,965 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T05:26:26,967 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-06T05:26:26,969 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T05:26:26,970 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35195
2024-12-06T05:26:26,972 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35195 connecting to ZooKeeper ensemble=127.0.0.1:49487
2024-12-06T05:26:26,973 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:26:26,976 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:26:26,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:351950x0, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T05:26:26,992 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:351950x0, quorum=127.0.0.1:49487, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:26:26,992 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35195-0x101a91af90e0001 connected
2024-12-06T05:26:26,995 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-06T05:26:27,002 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-06T05:26:27,004 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-06T05:26:27,008 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T05:26:27,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35195
2024-12-06T05:26:27,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35195
2024-12-06T05:26:27,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35195
2024-12-06T05:26:27,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35195
2024-12-06T05:26:27,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35195
2024-12-06T05:26:27,025 DEBUG [M:0;ec46afd49254:39099 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ec46afd49254:39099
2024-12-06T05:26:27,026 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ec46afd49254,39099,1733462786182
2024-12-06T05:26:27,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:26:27,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:26:27,046 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ec46afd49254,39099,1733462786182
2024-12-06T05:26:27,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:26:27,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-06T05:26:27,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:26:27,076 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-06T05:26:27,077 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ec46afd49254,39099,1733462786182 from backup master directory
2024-12-06T05:26:27,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ec46afd49254,39099,1733462786182
2024-12-06T05:26:27,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:26:27,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:26:27,086 WARN  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T05:26:27,086 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ec46afd49254,39099,1733462786182
2024-12-06T05:26:27,088 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-12-06T05:26:27,089 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-12-06T05:26:27,146 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/hbase.id] with ID: ca6bd0d9-9729-45f4-9a62-aeaa307b7eab
2024-12-06T05:26:27,146 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/.tmp/hbase.id
2024-12-06T05:26:27,157 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741826_1002 (size=42)
2024-12-06T05:26:27,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741826_1002 (size=42)
2024-12-06T05:26:27,159 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/.tmp/hbase.id]:[hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/hbase.id]
2024-12-06T05:26:27,204 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:26:27,209 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-06T05:26:27,227 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms.
2024-12-06T05:26:27,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:26:27,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:26:27,256 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741827_1003 (size=196)
2024-12-06T05:26:27,256 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741827_1003 (size=196)
2024-12-06T05:26:27,274 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T05:26:27,276 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-06T05:26:27,282 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:26:27,313 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741828_1004 (size=1189)
2024-12-06T05:26:27,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741828_1004 (size=1189)
2024-12-06T05:26:27,330 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store
2024-12-06T05:26:27,348 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741829_1005 (size=34)
2024-12-06T05:26:27,348 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741829_1005 (size=34)
2024-12-06T05:26:27,353 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-12-06T05:26:27,356 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:26:27,358 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T05:26:27,358 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:26:27,358 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:26:27,359 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T05:26:27,360 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:26:27,360 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:26:27,361 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733462787358Disabling compacts and flushes for region at 1733462787358Disabling writes for close at 1733462787359 (+1 ms)Writing region close event to WAL at 1733462787360 (+1 ms)Closed at 1733462787360
2024-12-06T05:26:27,363 WARN  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/.initializing
2024-12-06T05:26:27,364 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/WALs/ec46afd49254,39099,1733462786182
2024-12-06T05:26:27,387 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C39099%2C1733462786182, suffix=, logDir=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/WALs/ec46afd49254,39099,1733462786182, archiveDir=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/oldWALs, maxLogs=10
2024-12-06T05:26:27,398 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C39099%2C1733462786182.1733462787393
2024-12-06T05:26:27,418 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/WALs/ec46afd49254,39099,1733462786182/ec46afd49254%2C39099%2C1733462786182.1733462787393
2024-12-06T05:26:27,426 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37683:37683),(127.0.0.1/127.0.0.1:38339:38339)]
2024-12-06T05:26:27,427 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:26:27,427 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:26:27,429 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:26:27,430 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:26:27,463 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:26:27,485 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-06T05:26:27,488 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:26:27,490 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:26:27,490 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:26:27,494 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-06T05:26:27,494 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:26:27,495 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:26:27,495 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:26:27,498 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-06T05:26:27,498 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:26:27,499 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:26:27,499 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:26:27,501 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-06T05:26:27,502 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:26:27,503 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:26:27,503 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:26:27,507 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:26:27,508 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:26:27,515 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:26:27,515 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:26:27,519 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-06T05:26:27,523 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:26:27,527 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:26:27,529 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=700307, jitterRate=-0.10951431095600128}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-06T05:26:27,535 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733462787441Initializing all the Stores at 1733462787443 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462787444 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462787444Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462787445 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462787445Cleaning up temporary data from old regions at 1733462787516 (+71 ms)Region opened successfully at 1733462787534 (+18 ms)
2024-12-06T05:26:27,536 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-06T05:26:27,566 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c42c7c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0
2024-12-06T05:26:27,592 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-06T05:26:27,602 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-06T05:26:27,602 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-06T05:26:27,605 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-06T05:26:27,606 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-12-06T05:26:27,610 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-12-06T05:26:27,610 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-06T05:26:27,633 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-06T05:26:27,641 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-06T05:26:27,670 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-06T05:26:27,673 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-06T05:26:27,675 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-06T05:26:27,797 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-06T05:26:27,806 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-06T05:26:27,812 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-06T05:26:27,884 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-06T05:26:27,887 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-06T05:26:27,917 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-06T05:26:27,941 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-06T05:26:27,954 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-06T05:26:27,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T05:26:27,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T05:26:27,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:26:27,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:26:27,974 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ec46afd49254,39099,1733462786182, sessionid=0x101a91af90e0000, setting cluster-up flag (Was=false)
2024-12-06T05:26:28,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:26:28,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:26:28,043 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-06T05:26:28,047 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec46afd49254,39099,1733462786182
2024-12-06T05:26:28,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:26:28,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:26:28,106 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-06T05:26:28,111 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec46afd49254,39099,1733462786182
2024-12-06T05:26:28,120 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-06T05:26:28,183 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-06T05:26:28,191 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-06T05:26:28,197 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-06T05:26:28,202 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ec46afd49254,39099,1733462786182
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-06T05:26:28,209 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:26:28,209 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:26:28,210 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:26:28,210 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:26:28,210 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ec46afd49254:0, corePoolSize=10, maxPoolSize=10
2024-12-06T05:26:28,210 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:26:28,210 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ec46afd49254:0, corePoolSize=2, maxPoolSize=2
2024-12-06T05:26:28,210 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:26:28,212 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733462818211
2024-12-06T05:26:28,213 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-06T05:26:28,214 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-06T05:26:28,215 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.HRegionServer(746): ClusterId : ca6bd0d9-9729-45f4-9a62-aeaa307b7eab
2024-12-06T05:26:28,215 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:26:28,216 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-06T05:26:28,218 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-06T05:26:28,218 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-06T05:26:28,218 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-06T05:26:28,218 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-06T05:26:28,219 DEBUG [RS:0;ec46afd49254:35195 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-06T05:26:28,219 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:28,222 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:26:28,222 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-06T05:26:28,222 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-06T05:26:28,224 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-06T05:26:28,224 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-06T05:26:28,226 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-06T05:26:28,226 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-06T05:26:28,228 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462788227,5,FailOnTimeoutGroup]
2024-12-06T05:26:28,228 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462788228,5,FailOnTimeoutGroup]
2024-12-06T05:26:28,228 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:28,229 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-06T05:26:28,230 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:28,230 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:28,232 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741831_1007 (size=1321)
2024-12-06T05:26:28,232 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741831_1007 (size=1321)
2024-12-06T05:26:28,234 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-06T05:26:28,234 DEBUG [RS:0;ec46afd49254:35195 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-06T05:26:28,234 DEBUG [RS:0;ec46afd49254:35195 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-06T05:26:28,235 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1
2024-12-06T05:26:28,244 DEBUG [RS:0;ec46afd49254:35195 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-06T05:26:28,245 DEBUG [RS:0;ec46afd49254:35195 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a47d67b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0
2024-12-06T05:26:28,247 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741832_1008 (size=32)
2024-12-06T05:26:28,247 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741832_1008 (size=32)
2024-12-06T05:26:28,260 DEBUG [RS:0;ec46afd49254:35195 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ec46afd49254:35195
2024-12-06T05:26:28,264 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-06T05:26:28,264 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-06T05:26:28,264 DEBUG [RS:0;ec46afd49254:35195 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-06T05:26:28,266 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec46afd49254,39099,1733462786182 with port=35195, startcode=1733462786931
2024-12-06T05:26:28,276 DEBUG [RS:0;ec46afd49254:35195 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-06T05:26:28,326 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52971, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-12-06T05:26:28,331 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39099 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ec46afd49254,35195,1733462786931
2024-12-06T05:26:28,334 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39099 {}] master.ServerManager(517): Registering regionserver=ec46afd49254,35195,1733462786931
2024-12-06T05:26:28,345 DEBUG [RS:0;ec46afd49254:35195 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1
2024-12-06T05:26:28,345 DEBUG [RS:0;ec46afd49254:35195 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40559
2024-12-06T05:26:28,346 DEBUG [RS:0;ec46afd49254:35195 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-06T05:26:28,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T05:26:28,376 DEBUG [RS:0;ec46afd49254:35195 {}] zookeeper.ZKUtil(111): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec46afd49254,35195,1733462786931
2024-12-06T05:26:28,377 WARN  [RS:0;ec46afd49254:35195 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T05:26:28,377 INFO  [RS:0;ec46afd49254:35195 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:26:28,377 DEBUG [RS:0;ec46afd49254:35195 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931
2024-12-06T05:26:28,380 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec46afd49254,35195,1733462786931]
2024-12-06T05:26:28,406 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-06T05:26:28,418 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-06T05:26:28,423 INFO  [RS:0;ec46afd49254:35195 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T05:26:28,423 INFO  [RS:0;ec46afd49254:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:28,424 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-06T05:26:28,429 INFO  [RS:0;ec46afd49254:35195 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-06T05:26:28,430 INFO  [RS:0;ec46afd49254:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:28,430 DEBUG [RS:0;ec46afd49254:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:26:28,431 DEBUG [RS:0;ec46afd49254:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:26:28,431 DEBUG [RS:0;ec46afd49254:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:26:28,431 DEBUG [RS:0;ec46afd49254:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:26:28,431 DEBUG [RS:0;ec46afd49254:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:26:28,431 DEBUG [RS:0;ec46afd49254:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0, corePoolSize=2, maxPoolSize=2
2024-12-06T05:26:28,431 DEBUG [RS:0;ec46afd49254:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:26:28,431 DEBUG [RS:0;ec46afd49254:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:26:28,432 DEBUG [RS:0;ec46afd49254:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:26:28,432 DEBUG [RS:0;ec46afd49254:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:26:28,432 DEBUG [RS:0;ec46afd49254:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:26:28,432 DEBUG [RS:0;ec46afd49254:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:26:28,432 DEBUG [RS:0;ec46afd49254:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3
2024-12-06T05:26:28,432 DEBUG [RS:0;ec46afd49254:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3
2024-12-06T05:26:28,433 INFO  [RS:0;ec46afd49254:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:28,433 INFO  [RS:0;ec46afd49254:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:28,434 INFO  [RS:0;ec46afd49254:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:28,434 INFO  [RS:0;ec46afd49254:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:28,434 INFO  [RS:0;ec46afd49254:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:28,434 INFO  [RS:0;ec46afd49254:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,35195,1733462786931-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T05:26:28,450 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-06T05:26:28,451 INFO  [RS:0;ec46afd49254:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,35195,1733462786931-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:28,452 INFO  [RS:0;ec46afd49254:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:28,452 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.Replication(171): ec46afd49254,35195,1733462786931 started
2024-12-06T05:26:28,467 INFO  [RS:0;ec46afd49254:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:28,467 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.HRegionServer(1482): Serving as ec46afd49254,35195,1733462786931, RpcServer on ec46afd49254/172.17.0.3:35195, sessionid=0x101a91af90e0001
2024-12-06T05:26:28,468 DEBUG [RS:0;ec46afd49254:35195 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-06T05:26:28,468 DEBUG [RS:0;ec46afd49254:35195 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec46afd49254,35195,1733462786931
2024-12-06T05:26:28,468 DEBUG [RS:0;ec46afd49254:35195 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,35195,1733462786931'
2024-12-06T05:26:28,469 DEBUG [RS:0;ec46afd49254:35195 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-06T05:26:28,470 DEBUG [RS:0;ec46afd49254:35195 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-06T05:26:28,471 DEBUG [RS:0;ec46afd49254:35195 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-06T05:26:28,471 DEBUG [RS:0;ec46afd49254:35195 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-06T05:26:28,471 DEBUG [RS:0;ec46afd49254:35195 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec46afd49254,35195,1733462786931
2024-12-06T05:26:28,471 DEBUG [RS:0;ec46afd49254:35195 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,35195,1733462786931'
2024-12-06T05:26:28,471 DEBUG [RS:0;ec46afd49254:35195 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-06T05:26:28,472 DEBUG [RS:0;ec46afd49254:35195 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-06T05:26:28,472 DEBUG [RS:0;ec46afd49254:35195 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-06T05:26:28,473 INFO  [RS:0;ec46afd49254:35195 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-06T05:26:28,473 INFO  [RS:0;ec46afd49254:35195 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-06T05:26:28,587 INFO  [RS:0;ec46afd49254:35195 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C35195%2C1733462786931, suffix=, logDir=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931, archiveDir=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/oldWALs, maxLogs=32
2024-12-06T05:26:28,589 INFO  [RS:0;ec46afd49254:35195 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C35195%2C1733462786931.1733462788589
2024-12-06T05:26:28,598 INFO  [RS:0;ec46afd49254:35195 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462788589
2024-12-06T05:26:28,602 DEBUG [RS:0;ec46afd49254:35195 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38339:38339),(127.0.0.1/127.0.0.1:37683:37683)]
2024-12-06T05:26:28,649 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:26:28,653 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T05:26:28,658 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T05:26:28,658 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:26:28,660 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:26:28,660 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T05:26:28,662 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T05:26:28,662 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:26:28,663 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:26:28,663 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T05:26:28,666 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T05:26:28,666 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:26:28,666 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:26:28,667 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T05:26:28,669 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T05:26:28,669 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:26:28,670 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:26:28,670 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T05:26:28,671 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740
2024-12-06T05:26:28,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740
2024-12-06T05:26:28,675 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T05:26:28,675 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T05:26:28,676 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T05:26:28,679 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T05:26:28,683 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:26:28,684 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=718338, jitterRate=-0.08658653497695923}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T05:26:28,686 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733462788649Initializing all the Stores at 1733462788652 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462788652Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462788653 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462788653Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462788653Cleaning up temporary data from old regions at 1733462788675 (+22 ms)Region opened successfully at 1733462788686 (+11 ms)
2024-12-06T05:26:28,687 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T05:26:28,687 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T05:26:28,687 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T05:26:28,687 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T05:26:28,687 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T05:26:28,688 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T05:26:28,688 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733462788687Disabling compacts and flushes for region at 1733462788687Disabling writes for close at 1733462788687Writing region close event to WAL at 1733462788688 (+1 ms)Closed at 1733462788688
2024-12-06T05:26:28,692 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:26:28,692 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-06T05:26:28,698 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-06T05:26:28,706 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T05:26:28,708 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-06T05:26:28,863 DEBUG [ec46afd49254:39099 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-06T05:26:28,875 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ec46afd49254,35195,1733462786931
2024-12-06T05:26:28,881 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec46afd49254,35195,1733462786931, state=OPENING
2024-12-06T05:26:28,938 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-06T05:26:28,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:26:28,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:26:28,951 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:26:28,951 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:26:28,955 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T05:26:28,957 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ec46afd49254,35195,1733462786931}]
2024-12-06T05:26:29,138 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-06T05:26:29,141 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38845, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-06T05:26:29,151 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-06T05:26:29,152 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:26:29,155 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C35195%2C1733462786931.meta, suffix=.meta, logDir=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931, archiveDir=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/oldWALs, maxLogs=32
2024-12-06T05:26:29,157 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C35195%2C1733462786931.meta.1733462789156.meta
2024-12-06T05:26:29,164 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.meta.1733462789156.meta
2024-12-06T05:26:29,166 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37683:37683),(127.0.0.1/127.0.0.1:38339:38339)]
2024-12-06T05:26:29,168 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:26:29,170 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-06T05:26:29,172 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-06T05:26:29,178 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-06T05:26:29,183 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-06T05:26:29,184 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:26:29,185 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-06T05:26:29,185 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-06T05:26:29,188 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T05:26:29,190 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T05:26:29,190 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:26:29,191 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:26:29,191 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T05:26:29,192 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T05:26:29,193 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:26:29,193 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:26:29,194 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T05:26:29,195 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T05:26:29,195 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:26:29,196 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:26:29,196 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T05:26:29,197 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T05:26:29,197 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:26:29,198 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:26:29,198 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T05:26:29,200 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740
2024-12-06T05:26:29,203 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740
2024-12-06T05:26:29,205 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T05:26:29,205 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T05:26:29,206 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T05:26:29,209 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T05:26:29,211 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=768054, jitterRate=-0.023369133472442627}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T05:26:29,211 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-06T05:26:29,213 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733462789185Writing region info on filesystem at 1733462789186 (+1 ms)Initializing all the Stores at 1733462789188 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462789188Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462789188Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462789188Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462789188Cleaning up temporary data from old regions at 1733462789205 (+17 ms)Running coprocessor post-open hooks at 1733462789211 (+6 ms)Region opened successfully at 1733462789212 (+1 ms)
2024-12-06T05:26:29,220 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733462789130
2024-12-06T05:26:29,231 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-06T05:26:29,231 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-06T05:26:29,233 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ec46afd49254,35195,1733462786931
2024-12-06T05:26:29,235 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec46afd49254,35195,1733462786931, state=OPEN
2024-12-06T05:26:29,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T05:26:29,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T05:26:29,303 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:26:29,303 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:26:29,304 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ec46afd49254,35195,1733462786931
2024-12-06T05:26:29,309 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-06T05:26:29,309 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec46afd49254,35195,1733462786931 in 347 msec
2024-12-06T05:26:29,316 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-06T05:26:29,316 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 613 msec
2024-12-06T05:26:29,317 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:26:29,317 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-06T05:26:29,334 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T05:26:29,335 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec46afd49254,35195,1733462786931, seqNum=-1]
2024-12-06T05:26:29,352 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T05:26:29,355 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36447, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T05:26:29,374 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2250 sec
2024-12-06T05:26:29,374 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733462789374, completionTime=-1
2024-12-06T05:26:29,376 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-06T05:26:29,377 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-06T05:26:29,400 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-06T05:26:29,400 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733462849400
2024-12-06T05:26:29,400 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733462909400
2024-12-06T05:26:29,400 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec
2024-12-06T05:26:29,403 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,39099,1733462786182-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:29,403 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,39099,1733462786182-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:29,404 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,39099,1733462786182-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:29,405 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ec46afd49254:39099, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:29,406 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:29,407 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:29,413 DEBUG [master/ec46afd49254:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T05:26:29,434 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.348sec
2024-12-06T05:26:29,435 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-06T05:26:29,436 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-06T05:26:29,437 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-06T05:26:29,437 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-06T05:26:29,437 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-06T05:26:29,438 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,39099,1733462786182-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T05:26:29,438 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,39099,1733462786182-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-06T05:26:29,446 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-06T05:26:29,447 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-06T05:26:29,447 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,39099,1733462786182-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:26:29,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66cb686a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:26:29,546 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-12-06T05:26:29,546 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-12-06T05:26:29,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ec46afd49254,39099,-1 for getting cluster id
2024-12-06T05:26:29,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-06T05:26:29,562 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ca6bd0d9-9729-45f4-9a62-aeaa307b7eab'
2024-12-06T05:26:29,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-06T05:26:29,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ca6bd0d9-9729-45f4-9a62-aeaa307b7eab"

2024-12-06T05:26:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65f22587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:26:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ec46afd49254,39099,-1]
2024-12-06T05:26:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-06T05:26:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:26:29,579 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-06T05:26:29,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32fb6022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:26:29,583 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T05:26:29,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec46afd49254,35195,1733462786931, seqNum=-1]
2024-12-06T05:26:29,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T05:26:29,592 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T05:26:29,611 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ec46afd49254,39099,1733462786182
2024-12-06T05:26:29,611 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:26:29,617 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-06T05:26:29,622 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-06T05:26:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ec46afd49254,39099,1733462786182
2024-12-06T05:26:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@290bf544
2024-12-06T05:26:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-06T05:26:29,634 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-06T05:26:29,636 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39099 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-06T05:26:29,636 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39099 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-06T05:26:29,640 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39099 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T05:26:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39099 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-12-06T05:26:29,649 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-06T05:26:29,651 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39099 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-12-06T05:26:29,652 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:26:29,654 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-06T05:26:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39099 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T05:26:29,701 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741835_1011 (size=389)
2024-12-06T05:26:29,701 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741835_1011 (size=389)
2024-12-06T05:26:29,707 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b7256eebeb2cad440fd79ee8a9442daa, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1
2024-12-06T05:26:29,718 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741836_1012 (size=72)
2024-12-06T05:26:29,718 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741836_1012 (size=72)
2024-12-06T05:26:29,719 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:26:29,719 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing b7256eebeb2cad440fd79ee8a9442daa, disabling compactions & flushes
2024-12-06T05:26:29,719 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.
2024-12-06T05:26:29,719 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.
2024-12-06T05:26:29,719 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa. after waiting 0 ms
2024-12-06T05:26:29,720 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.
2024-12-06T05:26:29,720 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.
2024-12-06T05:26:29,720 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for b7256eebeb2cad440fd79ee8a9442daa:
Waiting for close lock at 1733462789719Disabling compacts and flushes for region at 1733462789719Disabling writes for close at 1733462789719Writing region close event to WAL at 1733462789720 (+1 ms)Closed at 1733462789720
2024-12-06T05:26:29,721 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-06T05:26:29,726 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1733462789722"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733462789722"}]},"ts":"1733462789722"}
2024-12-06T05:26:29,730 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-06T05:26:29,732 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-06T05:26:29,735 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733462789732"}]},"ts":"1733462789732"}
2024-12-06T05:26:29,739 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-12-06T05:26:29,741 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=b7256eebeb2cad440fd79ee8a9442daa, ASSIGN}]
2024-12-06T05:26:29,743 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=b7256eebeb2cad440fd79ee8a9442daa, ASSIGN
2024-12-06T05:26:29,745 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=b7256eebeb2cad440fd79ee8a9442daa, ASSIGN; state=OFFLINE, location=ec46afd49254,35195,1733462786931; forceNewPlan=false, retain=false
2024-12-06T05:26:29,897 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b7256eebeb2cad440fd79ee8a9442daa, regionState=OPENING, regionLocation=ec46afd49254,35195,1733462786931
2024-12-06T05:26:29,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=b7256eebeb2cad440fd79ee8a9442daa, ASSIGN because future has completed
2024-12-06T05:26:29,907 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7256eebeb2cad440fd79ee8a9442daa, server=ec46afd49254,35195,1733462786931}]
2024-12-06T05:26:30,077 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.
2024-12-06T05:26:30,077 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b7256eebeb2cad440fd79ee8a9442daa, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:26:30,078 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling b7256eebeb2cad440fd79ee8a9442daa
2024-12-06T05:26:30,078 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:26:30,078 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b7256eebeb2cad440fd79ee8a9442daa
2024-12-06T05:26:30,078 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b7256eebeb2cad440fd79ee8a9442daa
2024-12-06T05:26:30,081 INFO  [StoreOpener-b7256eebeb2cad440fd79ee8a9442daa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region b7256eebeb2cad440fd79ee8a9442daa 
2024-12-06T05:26:30,083 INFO  [StoreOpener-b7256eebeb2cad440fd79ee8a9442daa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7256eebeb2cad440fd79ee8a9442daa columnFamilyName info
2024-12-06T05:26:30,083 DEBUG [StoreOpener-b7256eebeb2cad440fd79ee8a9442daa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:26:30,084 INFO  [StoreOpener-b7256eebeb2cad440fd79ee8a9442daa-1 {}] regionserver.HStore(327): Store=b7256eebeb2cad440fd79ee8a9442daa/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:26:30,085 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b7256eebeb2cad440fd79ee8a9442daa
2024-12-06T05:26:30,086 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa
2024-12-06T05:26:30,087 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa
2024-12-06T05:26:30,088 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b7256eebeb2cad440fd79ee8a9442daa
2024-12-06T05:26:30,088 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b7256eebeb2cad440fd79ee8a9442daa
2024-12-06T05:26:30,091 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b7256eebeb2cad440fd79ee8a9442daa
2024-12-06T05:26:30,094 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:26:30,095 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b7256eebeb2cad440fd79ee8a9442daa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=789258, jitterRate=0.0035936832427978516}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-06T05:26:30,095 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7256eebeb2cad440fd79ee8a9442daa
2024-12-06T05:26:30,096 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b7256eebeb2cad440fd79ee8a9442daa:
Running coprocessor pre-open hook at 1733462790078Writing region info on filesystem at 1733462790078Initializing all the Stores at 1733462790080 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462790081 (+1 ms)Cleaning up temporary data from old regions at 1733462790088 (+7 ms)Running coprocessor post-open hooks at 1733462790095 (+7 ms)Region opened successfully at 1733462790096 (+1 ms)
2024-12-06T05:26:30,098 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa., pid=6, masterSystemTime=1733462790064
2024-12-06T05:26:30,102 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.
2024-12-06T05:26:30,102 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.
2024-12-06T05:26:30,103 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b7256eebeb2cad440fd79ee8a9442daa, regionState=OPEN, openSeqNum=2, regionLocation=ec46afd49254,35195,1733462786931
2024-12-06T05:26:30,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7256eebeb2cad440fd79ee8a9442daa, server=ec46afd49254,35195,1733462786931 because future has completed
2024-12-06T05:26:30,114 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-06T05:26:30,116 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b7256eebeb2cad440fd79ee8a9442daa, server=ec46afd49254,35195,1733462786931 in 203 msec
2024-12-06T05:26:30,118 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-06T05:26:30,118 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=b7256eebeb2cad440fd79ee8a9442daa, ASSIGN in 373 msec
2024-12-06T05:26:30,120 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-06T05:26:30,120 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733462790120"}]},"ts":"1733462790120"}
2024-12-06T05:26:30,124 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-12-06T05:26:30,126 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-06T05:26:30,130 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 484 msec
2024-12-06T05:26:34,702 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-06T05:26:34,748 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-06T05:26:34,749 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-12-06T05:26:36,506 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-06T05:26:36,507 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-06T05:26:36,511 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-06T05:26:36,511 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-06T05:26:36,514 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T05:26:36,514 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-06T05:26:36,515 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-06T05:26:36,515 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-06T05:26:39,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39099 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T05:26:39,710 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-12-06T05:26:39,716 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-12-06T05:26:39,722 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-12-06T05:26:39,723 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.
2024-12-06T05:26:39,724 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C35195%2C1733462786931.1733462799723
2024-12-06T05:26:39,775 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:26:39,776 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:26:39,776 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:26:39,776 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:26:39,776 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:26:39,777 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462788589 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462799723
2024-12-06T05:26:39,778 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38339:38339),(127.0.0.1/127.0.0.1:37683:37683)]
2024-12-06T05:26:39,779 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462788589 is not closed yet, will try archiving it next time
2024-12-06T05:26:39,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741833_1009 (size=451)
2024-12-06T05:26:39,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741833_1009 (size=451)
2024-12-06T05:26:39,783 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462788589 to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/oldWALs/ec46afd49254%2C35195%2C1733462786931.1733462788589
2024-12-06T05:26:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa., hostname=ec46afd49254,35195,1733462786931, seqNum=2]
2024-12-06T05:26:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35195 {}] regionserver.HRegion(8855): Flush requested on b7256eebeb2cad440fd79ee8a9442daa
2024-12-06T05:26:51,858 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7256eebeb2cad440fd79ee8a9442daa 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T05:26:51,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp/info/67df899e4ffc4651b284b56f6a5cee17 is 1080, key is row0001/info:/1733462799792/Put/seqid=0
2024-12-06T05:26:51,921 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741838_1014 (size=12509)
2024-12-06T05:26:51,921 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741838_1014 (size=12509)
2024-12-06T05:26:51,922 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp/info/67df899e4ffc4651b284b56f6a5cee17
2024-12-06T05:26:51,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp/info/67df899e4ffc4651b284b56f6a5cee17 as hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/67df899e4ffc4651b284b56f6a5cee17
2024-12-06T05:26:51,975 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/67df899e4ffc4651b284b56f6a5cee17, entries=7, sequenceid=11, filesize=12.2 K
2024-12-06T05:26:51,982 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for b7256eebeb2cad440fd79ee8a9442daa in 124ms, sequenceid=11, compaction requested=false
2024-12-06T05:26:51,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7256eebeb2cad440fd79ee8a9442daa:

2024-12-06T05:26:55,329 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T05:26:59,874 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C35195%2C1733462786931.1733462819874
2024-12-06T05:27:00,091 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 212 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK], DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK]]
2024-12-06T05:27:00,092 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:00,092 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:00,092 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:00,092 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:00,093 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:00,093 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462799723 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462819874
2024-12-06T05:27:00,094 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38339:38339),(127.0.0.1/127.0.0.1:37683:37683)]
2024-12-06T05:27:00,094 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462799723 is not closed yet, will try archiving it next time
2024-12-06T05:27:00,096 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741837_1013 (size=12399)
2024-12-06T05:27:00,096 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741837_1013 (size=12399)
2024-12-06T05:27:00,299 INFO  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK], DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK]]
2024-12-06T05:27:02,506 INFO  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK], DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK]]
2024-12-06T05:27:04,712 INFO  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK], DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK]]
2024-12-06T05:27:06,920 INFO  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK], DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK]]
2024-12-06T05:27:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35195 {}] regionserver.HRegion(8855): Flush requested on b7256eebeb2cad440fd79ee8a9442daa
2024-12-06T05:27:06,920 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7256eebeb2cad440fd79ee8a9442daa 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T05:27:07,124 INFO  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK], DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK]]
2024-12-06T05:27:07,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp/info/9ff65083f5584f3c8d69b89a5454134f is 1080, key is row0008/info:/1733462813857/Put/seqid=0
2024-12-06T05:27:07,145 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741840_1016 (size=12509)
2024-12-06T05:27:07,146 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741840_1016 (size=12509)
2024-12-06T05:27:07,146 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp/info/9ff65083f5584f3c8d69b89a5454134f
2024-12-06T05:27:07,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp/info/9ff65083f5584f3c8d69b89a5454134f as hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/9ff65083f5584f3c8d69b89a5454134f
2024-12-06T05:27:07,167 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/9ff65083f5584f3c8d69b89a5454134f, entries=7, sequenceid=21, filesize=12.2 K
2024-12-06T05:27:07,370 INFO  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK], DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK]]
2024-12-06T05:27:07,370 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for b7256eebeb2cad440fd79ee8a9442daa in 450ms, sequenceid=21, compaction requested=false
2024-12-06T05:27:07,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7256eebeb2cad440fd79ee8a9442daa:

2024-12-06T05:27:07,371 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-12-06T05:27:07,371 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:27:07,372 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/67df899e4ffc4651b284b56f6a5cee17 because midkey is the same as first or last row
2024-12-06T05:27:09,129 INFO  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1368): Slow sync cost: 203 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK], DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK]]
2024-12-06T05:27:09,449 INFO  [master/ec46afd49254:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-06T05:27:09,449 INFO  [master/ec46afd49254:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-06T05:27:11,337 INFO  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK], DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK]]
2024-12-06T05:27:11,342 WARN  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK], DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK]]
2024-12-06T05:27:11,344 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ec46afd49254%2C35195%2C1733462786931:(num 1733462819874) roll requested
2024-12-06T05:27:11,345 INFO  [regionserver/ec46afd49254:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C35195%2C1733462786931.1733462831345
2024-12-06T05:27:11,560 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 212 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK], DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK]]
2024-12-06T05:27:11,560 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:11,561 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:11,561 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:11,561 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:11,561 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:11,562 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462819874 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462831345
2024-12-06T05:27:11,563 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37683:37683),(127.0.0.1/127.0.0.1:38339:38339)]
2024-12-06T05:27:11,563 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462819874 is not closed yet, will try archiving it next time
2024-12-06T05:27:11,564 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462799723 to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/oldWALs/ec46afd49254%2C35195%2C1733462786931.1733462799723
2024-12-06T05:27:11,565 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741839_1015 (size=7739)
2024-12-06T05:27:11,566 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741839_1015 (size=7739)
2024-12-06T05:27:13,545 INFO  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK], DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK]]
2024-12-06T05:27:15,078 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region b7256eebeb2cad440fd79ee8a9442daa, had cached 0 bytes from a total of 25018
2024-12-06T05:27:15,752 INFO  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK], DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK]]
2024-12-06T05:27:17,957 INFO  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK], DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK]]
2024-12-06T05:27:20,165 INFO  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK], DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK]]
2024-12-06T05:27:22,170 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-06T05:27:22,171 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C35195%2C1733462786931.1733462842170
2024-12-06T05:27:25,330 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T05:27:27,189 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5011 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK], DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK]]
2024-12-06T05:27:27,192 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5011 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK], DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK]]
2024-12-06T05:27:27,192 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ec46afd49254%2C35195%2C1733462786931:(num 1733462842170) roll requested
2024-12-06T05:27:27,193 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:27,193 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:27,193 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:27,193 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:27,193 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:27,194 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462831345 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462842170
2024-12-06T05:27:27,195 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38339:38339),(127.0.0.1/127.0.0.1:37683:37683)]
2024-12-06T05:27:27,195 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462831345 is not closed yet, will try archiving it next time
2024-12-06T05:27:27,195 INFO  [regionserver/ec46afd49254:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C35195%2C1733462786931.1733462847195
2024-12-06T05:27:27,196 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741841_1017 (size=4753)
2024-12-06T05:27:27,197 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741841_1017 (size=4753)
2024-12-06T05:27:32,200 INFO  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5002 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK], DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK]]
2024-12-06T05:27:32,200 WARN  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5002 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK], DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK]]
2024-12-06T05:27:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35195 {}] regionserver.HRegion(8855): Flush requested on b7256eebeb2cad440fd79ee8a9442daa
2024-12-06T05:27:32,201 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7256eebeb2cad440fd79ee8a9442daa 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T05:27:32,210 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5010 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK], DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK]]
2024-12-06T05:27:32,210 WARN  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5010 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK], DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK]]
2024-12-06T05:27:34,202 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-06T05:27:37,206 INFO  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK], DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK]]
2024-12-06T05:27:37,206 WARN  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK], DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK]]
2024-12-06T05:27:37,207 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:37,207 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:37,207 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:37,208 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:37,208 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:37,209 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462842170 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462847195
2024-12-06T05:27:37,213 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37683:37683),(127.0.0.1/127.0.0.1:38339:38339)]
2024-12-06T05:27:37,213 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462842170 is not closed yet, will try archiving it next time
2024-12-06T05:27:37,213 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741842_1018 (size=1569)
2024-12-06T05:27:37,213 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ec46afd49254%2C35195%2C1733462786931:(num 1733462847195) roll requested
2024-12-06T05:27:37,214 INFO  [regionserver/ec46afd49254:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C35195%2C1733462786931.1733462857214
2024-12-06T05:27:37,214 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741842_1018 (size=1569)
2024-12-06T05:27:37,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp/info/c99839d3763242698e89fee192d21131 is 1080, key is row0015/info:/1733462828924/Put/seqid=0
2024-12-06T05:27:37,223 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741844_1020 (size=12509)
2024-12-06T05:27:37,224 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741844_1020 (size=12509)
2024-12-06T05:27:37,224 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp/info/c99839d3763242698e89fee192d21131
2024-12-06T05:27:37,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp/info/c99839d3763242698e89fee192d21131 as hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/c99839d3763242698e89fee192d21131
2024-12-06T05:27:37,245 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/c99839d3763242698e89fee192d21131, entries=7, sequenceid=31, filesize=12.2 K
2024-12-06T05:27:42,253 INFO  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5007 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK], DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK]]
2024-12-06T05:27:42,254 WARN  [FSHLog-0-hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1-prefix:ec46afd49254,35195,1733462786931 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5007 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK], DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK]]
2024-12-06T05:27:42,254 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for b7256eebeb2cad440fd79ee8a9442daa in 10053ms, sequenceid=31, compaction requested=true
2024-12-06T05:27:42,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7256eebeb2cad440fd79ee8a9442daa:

2024-12-06T05:27:42,254 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-12-06T05:27:42,254 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:27:42,254 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/67df899e4ffc4651b284b56f6a5cee17 because midkey is the same as first or last row
2024-12-06T05:27:42,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7256eebeb2cad440fd79ee8a9442daa:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T05:27:42,256 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5040 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK], DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK]]
2024-12-06T05:27:42,256 WARN  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5040 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36807,DS-430c88c7-adbd-4ffa-951d-c5b3de5989c2,DISK], DatanodeInfoWithStorage[127.0.0.1:36161,DS-3a5fc36a-9ac4-4332-8d4e-bffc6c6c1f8e,DISK]]
2024-12-06T05:27:42,257 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:42,257 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:42,257 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:42,257 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:42,257 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:42,258 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462847195 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462857214
2024-12-06T05:27:42,259 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T05:27:42,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:27:42,261 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741843_1019 (size=438)
2024-12-06T05:27:42,261 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741843_1019 (size=438)
2024-12-06T05:27:42,262 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462819874 to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/oldWALs/ec46afd49254%2C35195%2C1733462786931.1733462819874
2024-12-06T05:27:42,264 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37683:37683),(127.0.0.1/127.0.0.1:38339:38339)]
2024-12-06T05:27:42,264 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T05:27:42,264 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ec46afd49254%2C35195%2C1733462786931:(num 1733462862264) roll requested
2024-12-06T05:27:42,264 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C35195%2C1733462786931.1733462862264
2024-12-06T05:27:42,264 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462831345 to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/oldWALs/ec46afd49254%2C35195%2C1733462786931.1733462831345
2024-12-06T05:27:42,266 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462842170 to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/oldWALs/ec46afd49254%2C35195%2C1733462786931.1733462842170
2024-12-06T05:27:42,266 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.HStore(1541): b7256eebeb2cad440fd79ee8a9442daa/info is initiating minor compaction (all files)
2024-12-06T05:27:42,266 INFO  [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7256eebeb2cad440fd79ee8a9442daa/info in TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.
2024-12-06T05:27:42,267 INFO  [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/67df899e4ffc4651b284b56f6a5cee17, hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/9ff65083f5584f3c8d69b89a5454134f, hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/c99839d3763242698e89fee192d21131] into tmpdir=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp, totalSize=36.6 K
2024-12-06T05:27:42,267 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462847195 to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/oldWALs/ec46afd49254%2C35195%2C1733462786931.1733462847195
2024-12-06T05:27:42,268 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67df899e4ffc4651b284b56f6a5cee17, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733462799792
2024-12-06T05:27:42,269 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ff65083f5584f3c8d69b89a5454134f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1733462813857
2024-12-06T05:27:42,270 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] compactions.Compactor(225): Compacting c99839d3763242698e89fee192d21131, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733462828924
2024-12-06T05:27:42,277 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:42,277 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:42,277 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:42,278 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:42,278 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:42,278 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462857214 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462862264
2024-12-06T05:27:42,280 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741845_1021 (size=93)
2024-12-06T05:27:42,281 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741845_1021 (size=93)
2024-12-06T05:27:42,281 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462857214 to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/oldWALs/ec46afd49254%2C35195%2C1733462786931.1733462857214
2024-12-06T05:27:42,289 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38339:38339),(127.0.0.1/127.0.0.1:37683:37683)]
2024-12-06T05:27:42,290 INFO  [regionserver/ec46afd49254:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C35195%2C1733462786931.1733462862290
2024-12-06T05:27:42,301 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:42,301 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:42,302 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:42,302 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:42,302 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:27:42,302 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462862264 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462862290
2024-12-06T05:27:42,304 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38339:38339),(127.0.0.1/127.0.0.1:37683:37683)]
2024-12-06T05:27:42,304 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/WALs/ec46afd49254,35195,1733462786931/ec46afd49254%2C35195%2C1733462786931.1733462862264 is not closed yet, will try archiving it next time
2024-12-06T05:27:42,304 INFO  [RS:0;ec46afd49254:35195-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7256eebeb2cad440fd79ee8a9442daa#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T05:27:42,304 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741846_1022 (size=1258)
2024-12-06T05:27:42,304 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741846_1022 (size=1258)
2024-12-06T05:27:42,305 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp/info/4a7ac3440d504687b0250629c2088c37 is 1080, key is row0001/info:/1733462799792/Put/seqid=0
2024-12-06T05:27:42,311 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741848_1024 (size=27710)
2024-12-06T05:27:42,311 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741848_1024 (size=27710)
2024-12-06T05:27:42,321 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp/info/4a7ac3440d504687b0250629c2088c37 as hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/4a7ac3440d504687b0250629c2088c37
2024-12-06T05:27:42,337 INFO  [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7256eebeb2cad440fd79ee8a9442daa/info of b7256eebeb2cad440fd79ee8a9442daa into 4a7ac3440d504687b0250629c2088c37(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T05:27:42,337 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7256eebeb2cad440fd79ee8a9442daa:

2024-12-06T05:27:42,339 INFO  [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa., storeName=b7256eebeb2cad440fd79ee8a9442daa/info, priority=13, startTime=1733462862256; duration=0sec
2024-12-06T05:27:42,339 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-06T05:27:42,339 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:27:42,339 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/4a7ac3440d504687b0250629c2088c37 because midkey is the same as first or last row
2024-12-06T05:27:42,339 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-06T05:27:42,339 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:27:42,340 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/4a7ac3440d504687b0250629c2088c37 because midkey is the same as first or last row
2024-12-06T05:27:42,340 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-06T05:27:42,340 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:27:42,340 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/4a7ac3440d504687b0250629c2088c37 because midkey is the same as first or last row
2024-12-06T05:27:42,340 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:27:42,340 DEBUG [RS:0;ec46afd49254:35195-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7256eebeb2cad440fd79ee8a9442daa:info
2024-12-06T05:27:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35195 {}] regionserver.HRegion(8855): Flush requested on b7256eebeb2cad440fd79ee8a9442daa
2024-12-06T05:27:54,329 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7256eebeb2cad440fd79ee8a9442daa 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T05:27:54,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp/info/a871c131f3454031a72cbdee1716a441 is 1080, key is row0022/info:/1733462862291/Put/seqid=0
2024-12-06T05:27:54,347 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741849_1025 (size=12509)
2024-12-06T05:27:54,348 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741849_1025 (size=12509)
2024-12-06T05:27:54,348 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp/info/a871c131f3454031a72cbdee1716a441
2024-12-06T05:27:54,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp/info/a871c131f3454031a72cbdee1716a441 as hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/a871c131f3454031a72cbdee1716a441
2024-12-06T05:27:54,367 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/a871c131f3454031a72cbdee1716a441, entries=7, sequenceid=42, filesize=12.2 K
2024-12-06T05:27:54,369 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for b7256eebeb2cad440fd79ee8a9442daa in 39ms, sequenceid=42, compaction requested=false
2024-12-06T05:27:54,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7256eebeb2cad440fd79ee8a9442daa:

2024-12-06T05:27:54,369 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-12-06T05:27:54,369 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:27:54,369 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/4a7ac3440d504687b0250629c2088c37 because midkey is the same as first or last row
2024-12-06T05:27:55,330 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T05:28:00,079 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region b7256eebeb2cad440fd79ee8a9442daa, had cached 0 bytes from a total of 40219
2024-12-06T05:28:02,353 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-06T05:28:02,353 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T05:28:02,354 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:28:02,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:02,364 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:02,364 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-06T05:28:02,364 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-06T05:28:02,365 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=368815740, stopped=false
2024-12-06T05:28:02,365 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ec46afd49254,39099,1733462786182
2024-12-06T05:28:02,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T05:28:02,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T05:28:02,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:02,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:02,383 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T05:28:02,384 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T05:28:02,384 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:28:02,384 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:28:02,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:02,384 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:28:02,385 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ec46afd49254,35195,1733462786931' *****
2024-12-06T05:28:02,385 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-06T05:28:02,385 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-06T05:28:02,386 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-06T05:28:02,386 INFO  [RS:0;ec46afd49254:35195 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-06T05:28:02,386 INFO  [RS:0;ec46afd49254:35195 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-06T05:28:02,386 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.HRegionServer(3091): Received CLOSE for b7256eebeb2cad440fd79ee8a9442daa
2024-12-06T05:28:02,387 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.HRegionServer(959): stopping server ec46afd49254,35195,1733462786931
2024-12-06T05:28:02,387 INFO  [RS:0;ec46afd49254:35195 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T05:28:02,387 INFO  [RS:0;ec46afd49254:35195 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ec46afd49254:35195.
2024-12-06T05:28:02,387 DEBUG [RS:0;ec46afd49254:35195 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:28:02,387 DEBUG [RS:0;ec46afd49254:35195 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:02,387 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b7256eebeb2cad440fd79ee8a9442daa, disabling compactions & flushes
2024-12-06T05:28:02,387 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.
2024-12-06T05:28:02,387 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-06T05:28:02,388 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.
2024-12-06T05:28:02,388 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-06T05:28:02,388 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-06T05:28:02,388 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa. after waiting 0 ms
2024-12-06T05:28:02,388 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.
2024-12-06T05:28:02,388 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-06T05:28:02,388 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b7256eebeb2cad440fd79ee8a9442daa 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-12-06T05:28:02,388 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-06T05:28:02,388 DEBUG [RS:0;ec46afd49254:35195 {}] regionserver.HRegionServer(1325): Online Regions={b7256eebeb2cad440fd79ee8a9442daa=TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa., 1588230740=hbase:meta,,1.1588230740}
2024-12-06T05:28:02,388 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T05:28:02,389 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T05:28:02,389 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T05:28:02,389 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T05:28:02,389 DEBUG [RS:0;ec46afd49254:35195 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, b7256eebeb2cad440fd79ee8a9442daa
2024-12-06T05:28:02,389 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T05:28:02,389 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-12-06T05:28:02,395 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp/info/4b4259f346eb4670b7cd96d9e7d91c1e is 1080, key is row0029/info:/1733462876333/Put/seqid=0
2024-12-06T05:28:02,404 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741850_1026 (size=8193)
2024-12-06T05:28:02,404 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741850_1026 (size=8193)
2024-12-06T05:28:02,405 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp/info/4b4259f346eb4670b7cd96d9e7d91c1e
2024-12-06T05:28:02,412 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740/.tmp/info/4a39585b84e84353a92d9b9bf1c4fb65 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa./info:regioninfo/1733462790103/Put/seqid=0
2024-12-06T05:28:02,416 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/.tmp/info/4b4259f346eb4670b7cd96d9e7d91c1e as hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/4b4259f346eb4670b7cd96d9e7d91c1e
2024-12-06T05:28:02,423 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741851_1027 (size=7016)
2024-12-06T05:28:02,423 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741851_1027 (size=7016)
2024-12-06T05:28:02,424 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740/.tmp/info/4a39585b84e84353a92d9b9bf1c4fb65
2024-12-06T05:28:02,425 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/4b4259f346eb4670b7cd96d9e7d91c1e, entries=3, sequenceid=48, filesize=8.0 K
2024-12-06T05:28:02,427 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for b7256eebeb2cad440fd79ee8a9442daa in 38ms, sequenceid=48, compaction requested=true
2024-12-06T05:28:02,427 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/67df899e4ffc4651b284b56f6a5cee17, hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/9ff65083f5584f3c8d69b89a5454134f, hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/c99839d3763242698e89fee192d21131] to archive
2024-12-06T05:28:02,431 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-06T05:28:02,434 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/67df899e4ffc4651b284b56f6a5cee17 to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/archive/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/67df899e4ffc4651b284b56f6a5cee17
2024-12-06T05:28:02,437 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/9ff65083f5584f3c8d69b89a5454134f to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/archive/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/9ff65083f5584f3c8d69b89a5454134f
2024-12-06T05:28:02,439 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/c99839d3763242698e89fee192d21131 to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/archive/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/info/c99839d3763242698e89fee192d21131
2024-12-06T05:28:02,440 INFO  [regionserver/ec46afd49254:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T05:28:02,456 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740/.tmp/ns/765cc9f1be8c427fbbbd445935fbe7f3 is 43, key is default/ns:d/1733462789359/Put/seqid=0
2024-12-06T05:28:02,453 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ec46afd49254:39099 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-06T05:28:02,458 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [67df899e4ffc4651b284b56f6a5cee17=12509, 9ff65083f5584f3c8d69b89a5454134f=12509, c99839d3763242698e89fee192d21131=12509]
2024-12-06T05:28:02,466 INFO  [regionserver/ec46afd49254:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-06T05:28:02,466 INFO  [regionserver/ec46afd49254:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-06T05:28:02,468 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/default/TestLogRolling-testSlowSyncLogRolling/b7256eebeb2cad440fd79ee8a9442daa/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-12-06T05:28:02,468 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741852_1028 (size=5153)
2024-12-06T05:28:02,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741852_1028 (size=5153)
2024-12-06T05:28:02,470 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740/.tmp/ns/765cc9f1be8c427fbbbd445935fbe7f3
2024-12-06T05:28:02,471 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.
2024-12-06T05:28:02,471 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b7256eebeb2cad440fd79ee8a9442daa:
Waiting for close lock at 1733462882387Running coprocessor pre-close hooks at 1733462882387Disabling compacts and flushes for region at 1733462882387Disabling writes for close at 1733462882388 (+1 ms)Obtaining lock to block concurrent updates at 1733462882388Preparing flush snapshotting stores in b7256eebeb2cad440fd79ee8a9442daa at 1733462882388Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1733462882388Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa. at 1733462882390 (+2 ms)Flushing b7256eebeb2cad440fd79ee8a9442daa/info: creating writer at 1733462882390Flushing b7256eebeb2cad440fd79ee8a9442daa/info: appending metadata at 1733462882395 (+5 ms)Flushing b7256eebeb2cad440fd79ee8a9442daa/info: closing flushed file at 1733462882395Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@668d2267: reopening flushed file at 1733462882414 (+19 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for b7256eebeb2cad440fd79ee8a9442daa in 38ms, sequenceid=48, compaction requested=true at 1733462882427 (+13 ms)Writing region close event to WAL at 1733462882459 (+32 ms)Running coprocessor post-close hooks at 1733462882469 (+10 ms)Closed at 1733462882470 (+1 ms)
2024-12-06T05:28:02,471 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1733462789636.b7256eebeb2cad440fd79ee8a9442daa.
2024-12-06T05:28:02,494 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740/.tmp/table/f9728758583049a3a53fbac8b5ae6614 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1733462790120/Put/seqid=0
2024-12-06T05:28:02,499 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741853_1029 (size=5396)
2024-12-06T05:28:02,500 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741853_1029 (size=5396)
2024-12-06T05:28:02,500 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740/.tmp/table/f9728758583049a3a53fbac8b5ae6614
2024-12-06T05:28:02,509 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740/.tmp/info/4a39585b84e84353a92d9b9bf1c4fb65 as hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740/info/4a39585b84e84353a92d9b9bf1c4fb65
2024-12-06T05:28:02,517 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740/info/4a39585b84e84353a92d9b9bf1c4fb65, entries=10, sequenceid=11, filesize=6.9 K
2024-12-06T05:28:02,518 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740/.tmp/ns/765cc9f1be8c427fbbbd445935fbe7f3 as hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740/ns/765cc9f1be8c427fbbbd445935fbe7f3
2024-12-06T05:28:02,525 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740/ns/765cc9f1be8c427fbbbd445935fbe7f3, entries=2, sequenceid=11, filesize=5.0 K
2024-12-06T05:28:02,526 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740/.tmp/table/f9728758583049a3a53fbac8b5ae6614 as hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740/table/f9728758583049a3a53fbac8b5ae6614
2024-12-06T05:28:02,534 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740/table/f9728758583049a3a53fbac8b5ae6614, entries=2, sequenceid=11, filesize=5.3 K
2024-12-06T05:28:02,536 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 147ms, sequenceid=11, compaction requested=false
2024-12-06T05:28:02,541 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-06T05:28:02,542 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T05:28:02,542 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T05:28:02,543 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733462882388Running coprocessor pre-close hooks at 1733462882388Disabling compacts and flushes for region at 1733462882388Disabling writes for close at 1733462882389 (+1 ms)Obtaining lock to block concurrent updates at 1733462882389Preparing flush snapshotting stores in 1588230740 at 1733462882389Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1733462882390 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733462882391 (+1 ms)Flushing 1588230740/info: creating writer at 1733462882391Flushing 1588230740/info: appending metadata at 1733462882411 (+20 ms)Flushing 1588230740/info: closing flushed file at 1733462882411Flushing 1588230740/ns: creating writer at 1733462882435 (+24 ms)Flushing 1588230740/ns: appending metadata at 1733462882455 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1733462882455Flushing 1588230740/table: creating writer at 1733462882478 (+23 ms)Flushing 1588230740/table: appending metadata at 1733462882493 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733462882493Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e0eb3c0: reopening flushed file at 1733462882508 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7388bac8: reopening flushed file at 1733462882517 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35827c4c: reopening flushed file at 1733462882525 (+8 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 147ms, sequenceid=11, compaction requested=false at 1733462882536 (+11 ms)Writing region close event to WAL at 1733462882537 (+1 ms)Running coprocessor post-close hooks at 1733462882542 (+5 ms)Closed at 1733462882542
2024-12-06T05:28:02,543 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-06T05:28:02,589 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.HRegionServer(976): stopping server ec46afd49254,35195,1733462786931; all regions closed.
2024-12-06T05:28:02,592 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:02,592 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:02,592 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:02,593 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:02,593 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:02,597 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741834_1010 (size=3066)
2024-12-06T05:28:02,598 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741834_1010 (size=3066)
2024-12-06T05:28:02,603 DEBUG [RS:0;ec46afd49254:35195 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/oldWALs
2024-12-06T05:28:02,603 INFO  [RS:0;ec46afd49254:35195 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ec46afd49254%2C35195%2C1733462786931.meta:.meta(num 1733462789156)
2024-12-06T05:28:02,603 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:02,604 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:02,604 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:02,604 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:02,604 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:02,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741847_1023 (size=12695)
2024-12-06T05:28:02,607 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741847_1023 (size=12695)
2024-12-06T05:28:02,610 DEBUG [RS:0;ec46afd49254:35195 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/oldWALs
2024-12-06T05:28:02,611 INFO  [RS:0;ec46afd49254:35195 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ec46afd49254%2C35195%2C1733462786931:(num 1733462862290)
2024-12-06T05:28:02,611 DEBUG [RS:0;ec46afd49254:35195 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:02,611 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T05:28:02,611 INFO  [RS:0;ec46afd49254:35195 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T05:28:02,611 INFO  [RS:0;ec46afd49254:35195 {}] hbase.ChoreService(370): Chore service for: regionserver/ec46afd49254:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-06T05:28:02,611 INFO  [RS:0;ec46afd49254:35195 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T05:28:02,611 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T05:28:02,612 INFO  [RS:0;ec46afd49254:35195 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35195
2024-12-06T05:28:02,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T05:28:02,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec46afd49254,35195,1733462786931
2024-12-06T05:28:02,636 INFO  [RS:0;ec46afd49254:35195 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T05:28:02,637 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec46afd49254,35195,1733462786931]
2024-12-06T05:28:02,678 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ec46afd49254,35195,1733462786931 already deleted, retry=false
2024-12-06T05:28:02,679 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ec46afd49254,35195,1733462786931 expired; onlineServers=0
2024-12-06T05:28:02,679 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ec46afd49254,39099,1733462786182' *****
2024-12-06T05:28:02,679 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-06T05:28:02,680 INFO  [M:0;ec46afd49254:39099 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T05:28:02,680 INFO  [M:0;ec46afd49254:39099 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T05:28:02,680 DEBUG [M:0;ec46afd49254:39099 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-06T05:28:02,681 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-06T05:28:02,681 DEBUG [M:0;ec46afd49254:39099 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-06T05:28:02,681 DEBUG [master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462788227 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462788227,5,FailOnTimeoutGroup]
2024-12-06T05:28:02,681 DEBUG [master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462788228 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462788228,5,FailOnTimeoutGroup]
2024-12-06T05:28:02,682 INFO  [M:0;ec46afd49254:39099 {}] hbase.ChoreService(370): Chore service for: master/ec46afd49254:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-06T05:28:02,682 INFO  [M:0;ec46afd49254:39099 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T05:28:02,682 DEBUG [M:0;ec46afd49254:39099 {}] master.HMaster(1795): Stopping service threads
2024-12-06T05:28:02,683 INFO  [M:0;ec46afd49254:39099 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-06T05:28:02,683 INFO  [M:0;ec46afd49254:39099 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T05:28:02,684 INFO  [M:0;ec46afd49254:39099 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-06T05:28:02,685 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-06T05:28:02,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-06T05:28:02,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:02,699 DEBUG [M:0;ec46afd49254:39099 {}] zookeeper.ZKUtil(347): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-06T05:28:02,699 WARN  [M:0;ec46afd49254:39099 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-06T05:28:02,700 INFO  [M:0;ec46afd49254:39099 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/.lastflushedseqids
2024-12-06T05:28:02,717 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741854_1030 (size=130)
2024-12-06T05:28:02,717 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741854_1030 (size=130)
2024-12-06T05:28:02,718 INFO  [M:0;ec46afd49254:39099 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-06T05:28:02,718 INFO  [M:0;ec46afd49254:39099 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-06T05:28:02,718 DEBUG [M:0;ec46afd49254:39099 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T05:28:02,718 INFO  [M:0;ec46afd49254:39099 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:02,718 DEBUG [M:0;ec46afd49254:39099 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:02,718 DEBUG [M:0;ec46afd49254:39099 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T05:28:02,718 DEBUG [M:0;ec46afd49254:39099 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:02,718 INFO  [M:0;ec46afd49254:39099 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.01 KB heapSize=29.18 KB
2024-12-06T05:28:02,742 DEBUG [M:0;ec46afd49254:39099 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d777eddec7b6480bbf88e982ece16193 is 82, key is hbase:meta,,1/info:regioninfo/1733462789233/Put/seqid=0
2024-12-06T05:28:02,749 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741855_1031 (size=5672)
2024-12-06T05:28:02,750 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741855_1031 (size=5672)
2024-12-06T05:28:02,750 INFO  [M:0;ec46afd49254:39099 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d777eddec7b6480bbf88e982ece16193
2024-12-06T05:28:02,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:28:02,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x101a91af90e0001, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:28:02,758 INFO  [RS:0;ec46afd49254:35195 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T05:28:02,758 INFO  [RS:0;ec46afd49254:35195 {}] regionserver.HRegionServer(1031): Exiting; stopping=ec46afd49254,35195,1733462786931; zookeeper connection closed.
2024-12-06T05:28:02,758 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@31dfbdaf {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@31dfbdaf
2024-12-06T05:28:02,759 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-06T05:28:02,775 DEBUG [M:0;ec46afd49254:39099 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0dc14728ca15415189773e36288acb29 is 765, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733462790128/Put/seqid=0
2024-12-06T05:28:02,782 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741856_1032 (size=6246)
2024-12-06T05:28:02,782 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741856_1032 (size=6246)
2024-12-06T05:28:02,782 INFO  [M:0;ec46afd49254:39099 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.41 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0dc14728ca15415189773e36288acb29
2024-12-06T05:28:02,788 INFO  [M:0;ec46afd49254:39099 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0dc14728ca15415189773e36288acb29
2024-12-06T05:28:02,804 DEBUG [M:0;ec46afd49254:39099 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/29a3764ca5064bc689ed93afd9395686 is 69, key is ec46afd49254,35195,1733462786931/rs:state/1733462788336/Put/seqid=0
2024-12-06T05:28:02,810 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741857_1033 (size=5156)
2024-12-06T05:28:02,810 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741857_1033 (size=5156)
2024-12-06T05:28:02,811 INFO  [M:0;ec46afd49254:39099 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/29a3764ca5064bc689ed93afd9395686
2024-12-06T05:28:02,842 DEBUG [M:0;ec46afd49254:39099 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3fd3a9da1bd74d83b57df449bfc4533f is 52, key is load_balancer_on/state:d/1733462789614/Put/seqid=0
2024-12-06T05:28:02,848 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741858_1034 (size=5056)
2024-12-06T05:28:02,848 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741858_1034 (size=5056)
2024-12-06T05:28:02,849 INFO  [M:0;ec46afd49254:39099 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3fd3a9da1bd74d83b57df449bfc4533f
2024-12-06T05:28:02,859 DEBUG [M:0;ec46afd49254:39099 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d777eddec7b6480bbf88e982ece16193 as hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d777eddec7b6480bbf88e982ece16193
2024-12-06T05:28:02,868 INFO  [M:0;ec46afd49254:39099 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d777eddec7b6480bbf88e982ece16193, entries=8, sequenceid=59, filesize=5.5 K
2024-12-06T05:28:02,869 DEBUG [M:0;ec46afd49254:39099 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0dc14728ca15415189773e36288acb29 as hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0dc14728ca15415189773e36288acb29
2024-12-06T05:28:02,878 INFO  [M:0;ec46afd49254:39099 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0dc14728ca15415189773e36288acb29
2024-12-06T05:28:02,878 INFO  [M:0;ec46afd49254:39099 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0dc14728ca15415189773e36288acb29, entries=6, sequenceid=59, filesize=6.1 K
2024-12-06T05:28:02,880 DEBUG [M:0;ec46afd49254:39099 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/29a3764ca5064bc689ed93afd9395686 as hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/29a3764ca5064bc689ed93afd9395686
2024-12-06T05:28:02,887 INFO  [M:0;ec46afd49254:39099 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/29a3764ca5064bc689ed93afd9395686, entries=1, sequenceid=59, filesize=5.0 K
2024-12-06T05:28:02,889 DEBUG [M:0;ec46afd49254:39099 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3fd3a9da1bd74d83b57df449bfc4533f as hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/3fd3a9da1bd74d83b57df449bfc4533f
2024-12-06T05:28:02,896 INFO  [M:0;ec46afd49254:39099 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/3fd3a9da1bd74d83b57df449bfc4533f, entries=1, sequenceid=59, filesize=4.9 K
2024-12-06T05:28:02,898 INFO  [M:0;ec46afd49254:39099 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.01 KB/23564, heapSize ~29.12 KB/29816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 180ms, sequenceid=59, compaction requested=false
2024-12-06T05:28:02,900 INFO  [M:0;ec46afd49254:39099 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:02,900 DEBUG [M:0;ec46afd49254:39099 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733462882718Disabling compacts and flushes for region at 1733462882718Disabling writes for close at 1733462882718Obtaining lock to block concurrent updates at 1733462882718Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733462882718Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23564, getHeapSize=29816, getOffHeapSize=0, getCellsCount=70 at 1733462882719 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733462882720 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733462882720Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733462882742 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733462882742Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733462882756 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733462882774 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733462882774Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733462882789 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733462882804 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733462882804Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733462882819 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733462882841 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733462882841Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68609b12: reopening flushed file at 1733462882858 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1306f75: reopening flushed file at 1733462882868 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@172f1f40: reopening flushed file at 1733462882878 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1af9d0ee: reopening flushed file at 1733462882888 (+10 ms)Finished flush of dataSize ~23.01 KB/23564, heapSize ~29.12 KB/29816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 180ms, sequenceid=59, compaction requested=false at 1733462882898 (+10 ms)Writing region close event to WAL at 1733462882899 (+1 ms)Closed at 1733462882899
2024-12-06T05:28:02,900 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:02,901 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:02,901 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:02,901 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:02,901 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:02,903 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741830_1006 (size=27961)
2024-12-06T05:28:02,903 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36807 is added to blk_1073741830_1006 (size=27961)
2024-12-06T05:28:02,904 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T05:28:02,904 INFO  [M:0;ec46afd49254:39099 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-06T05:28:02,904 INFO  [M:0;ec46afd49254:39099 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39099
2024-12-06T05:28:02,904 INFO  [M:0;ec46afd49254:39099 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T05:28:03,018 INFO  [M:0;ec46afd49254:39099 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T05:28:03,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:28:03,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39099-0x101a91af90e0000, quorum=127.0.0.1:49487, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:28:03,022 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:03,024 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:28:03,024 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:28:03,025 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:28:03,025 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/hadoop.log.dir/,STOPPED}
2024-12-06T05:28:03,027 WARN  [BP-225256256-172.17.0.3-1733462782207 heartbeating to localhost/127.0.0.1:40559 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:28:03,027 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:28:03,027 WARN  [BP-225256256-172.17.0.3-1733462782207 heartbeating to localhost/127.0.0.1:40559 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-225256256-172.17.0.3-1733462782207 (Datanode Uuid 3981a4fa-ccc0-4677-9f17-d0559b473553) service to localhost/127.0.0.1:40559
2024-12-06T05:28:03,027 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:28:03,028 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/cluster_5e02bba0-7a73-8521-fbdd-f73238a7d072/data/data3/current/BP-225256256-172.17.0.3-1733462782207 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:03,029 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/cluster_5e02bba0-7a73-8521-fbdd-f73238a7d072/data/data4/current/BP-225256256-172.17.0.3-1733462782207 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:03,029 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:28:03,032 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:03,032 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:28:03,032 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:28:03,033 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:28:03,033 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/hadoop.log.dir/,STOPPED}
2024-12-06T05:28:03,034 WARN  [BP-225256256-172.17.0.3-1733462782207 heartbeating to localhost/127.0.0.1:40559 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:28:03,034 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:28:03,034 WARN  [BP-225256256-172.17.0.3-1733462782207 heartbeating to localhost/127.0.0.1:40559 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-225256256-172.17.0.3-1733462782207 (Datanode Uuid d385fa8c-62ea-4020-9b21-63c6656c2f8e) service to localhost/127.0.0.1:40559
2024-12-06T05:28:03,034 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:28:03,035 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/cluster_5e02bba0-7a73-8521-fbdd-f73238a7d072/data/data1/current/BP-225256256-172.17.0.3-1733462782207 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:03,035 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/cluster_5e02bba0-7a73-8521-fbdd-f73238a7d072/data/data2/current/BP-225256256-172.17.0.3-1733462782207 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:03,036 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:28:03,047 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T05:28:03,048 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:28:03,048 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:28:03,048 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:28:03,048 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/hadoop.log.dir/,STOPPED}
2024-12-06T05:28:03,056 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-06T05:28:03,086 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-06T05:28:03,094 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=78 (was 12)
Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/ec46afd49254:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40559 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40559 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40559 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: master/ec46afd49254:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:40559
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40559
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40559
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:40559
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40559
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: master/ec46afd49254:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@5e1d09c6
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=404 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 151), ProcessCount=11 (was 11), AvailableMemoryMB=7641 (was 8403)
2024-12-06T05:28:03,100 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=79, OpenFileDescriptor=404, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=7640
2024-12-06T05:28:03,100 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-06T05:28:03,101 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/hadoop.log.dir so I do NOT create it in target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57
2024-12-06T05:28:03,101 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0152b7e7-d03e-b3da-4c0e-50ae5deadd63/hadoop.tmp.dir so I do NOT create it in target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57
2024-12-06T05:28:03,101 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/cluster_4b9a14b3-6a40-37da-1c8a-68406eef4d27, deleteOnExit=true
2024-12-06T05:28:03,101 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-06T05:28:03,101 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/test.cache.data in system properties and HBase conf
2024-12-06T05:28:03,101 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/hadoop.tmp.dir in system properties and HBase conf
2024-12-06T05:28:03,101 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/hadoop.log.dir in system properties and HBase conf
2024-12-06T05:28:03,101 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-06T05:28:03,101 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-06T05:28:03,102 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-06T05:28:03,102 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-06T05:28:03,102 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-06T05:28:03,102 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-06T05:28:03,102 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-06T05:28:03,102 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T05:28:03,102 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-06T05:28:03,102 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-06T05:28:03,102 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T05:28:03,103 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T05:28:03,103 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-06T05:28:03,103 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/nfs.dump.dir in system properties and HBase conf
2024-12-06T05:28:03,103 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/java.io.tmpdir in system properties and HBase conf
2024-12-06T05:28:03,103 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T05:28:03,103 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-06T05:28:03,103 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-06T05:28:03,117 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T05:28:03,423 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:28:03,428 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:28:03,429 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:28:03,429 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:28:03,429 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T05:28:03,433 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:28:03,433 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@737d6115{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:28:03,433 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@641eaf99{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:28:03,525 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2a7d11e5{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/java.io.tmpdir/jetty-localhost-46349-hadoop-hdfs-3_4_1-tests_jar-_-any-2117593886018944795/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T05:28:03,526 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5d4bdc00{HTTP/1.1, (http/1.1)}{localhost:46349}
2024-12-06T05:28:03,526 INFO  [Time-limited test {}] server.Server(415): Started @103137ms
2024-12-06T05:28:03,537 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T05:28:03,789 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:28:03,793 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:28:03,793 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:28:03,793 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:28:03,793 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T05:28:03,794 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b58749b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:28:03,794 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1aa9c156{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:28:03,887 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@66b9e907{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/java.io.tmpdir/jetty-localhost-42999-hadoop-hdfs-3_4_1-tests_jar-_-any-9568714158684740212/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:03,887 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@51f8cb4f{HTTP/1.1, (http/1.1)}{localhost:42999}
2024-12-06T05:28:03,887 INFO  [Time-limited test {}] server.Server(415): Started @103498ms
2024-12-06T05:28:03,889 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:28:03,927 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:28:03,930 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:28:03,931 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:28:03,931 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:28:03,931 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T05:28:03,932 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a742c1f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:28:03,932 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d5e070a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:28:04,021 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c17f9e5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/java.io.tmpdir/jetty-localhost-35395-hadoop-hdfs-3_4_1-tests_jar-_-any-8344355661664126872/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:04,022 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@669fbf01{HTTP/1.1, (http/1.1)}{localhost:35395}
2024-12-06T05:28:04,022 INFO  [Time-limited test {}] server.Server(415): Started @103633ms
2024-12-06T05:28:04,024 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:28:05,405 WARN  [Thread-454 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/cluster_4b9a14b3-6a40-37da-1c8a-68406eef4d27/data/data2/current/BP-1692227801-172.17.0.3-1733462883129/current, will proceed with Du for space computation calculation, 
2024-12-06T05:28:05,405 WARN  [Thread-453 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/cluster_4b9a14b3-6a40-37da-1c8a-68406eef4d27/data/data1/current/BP-1692227801-172.17.0.3-1733462883129/current, will proceed with Du for space computation calculation, 
2024-12-06T05:28:05,419 WARN  [Thread-417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:28:05,422 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb024160d0eee24b1 with lease ID 0x315ab9ea1bed8187: Processing first storage report for DS-31363a42-19a2-41ff-a6ea-baebb44f07f7 from datanode DatanodeRegistration(127.0.0.1:41205, datanodeUuid=f1551871-761a-491b-beca-c9ef99893f65, infoPort=39751, infoSecurePort=0, ipcPort=42403, storageInfo=lv=-57;cid=testClusterID;nsid=1340258790;c=1733462883129)
2024-12-06T05:28:05,422 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb024160d0eee24b1 with lease ID 0x315ab9ea1bed8187: from storage DS-31363a42-19a2-41ff-a6ea-baebb44f07f7 node DatanodeRegistration(127.0.0.1:41205, datanodeUuid=f1551871-761a-491b-beca-c9ef99893f65, infoPort=39751, infoSecurePort=0, ipcPort=42403, storageInfo=lv=-57;cid=testClusterID;nsid=1340258790;c=1733462883129), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-06T05:28:05,422 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb024160d0eee24b1 with lease ID 0x315ab9ea1bed8187: Processing first storage report for DS-aeb85d7b-03d2-4c0f-b5a5-52e7b4b81d65 from datanode DatanodeRegistration(127.0.0.1:41205, datanodeUuid=f1551871-761a-491b-beca-c9ef99893f65, infoPort=39751, infoSecurePort=0, ipcPort=42403, storageInfo=lv=-57;cid=testClusterID;nsid=1340258790;c=1733462883129)
2024-12-06T05:28:05,422 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb024160d0eee24b1 with lease ID 0x315ab9ea1bed8187: from storage DS-aeb85d7b-03d2-4c0f-b5a5-52e7b4b81d65 node DatanodeRegistration(127.0.0.1:41205, datanodeUuid=f1551871-761a-491b-beca-c9ef99893f65, infoPort=39751, infoSecurePort=0, ipcPort=42403, storageInfo=lv=-57;cid=testClusterID;nsid=1340258790;c=1733462883129), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:28:05,625 WARN  [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/cluster_4b9a14b3-6a40-37da-1c8a-68406eef4d27/data/data3/current/BP-1692227801-172.17.0.3-1733462883129/current, will proceed with Du for space computation calculation, 
2024-12-06T05:28:05,625 WARN  [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/cluster_4b9a14b3-6a40-37da-1c8a-68406eef4d27/data/data4/current/BP-1692227801-172.17.0.3-1733462883129/current, will proceed with Du for space computation calculation, 
2024-12-06T05:28:05,646 WARN  [Thread-440 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:28:05,649 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x270de150b695e1d1 with lease ID 0x315ab9ea1bed8188: Processing first storage report for DS-fc1c754a-7f80-4cdc-9a22-3f28cb8f8217 from datanode DatanodeRegistration(127.0.0.1:33941, datanodeUuid=d82ec060-f13b-41ba-88a0-0486bb38a0a9, infoPort=38427, infoSecurePort=0, ipcPort=45839, storageInfo=lv=-57;cid=testClusterID;nsid=1340258790;c=1733462883129)
2024-12-06T05:28:05,649 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x270de150b695e1d1 with lease ID 0x315ab9ea1bed8188: from storage DS-fc1c754a-7f80-4cdc-9a22-3f28cb8f8217 node DatanodeRegistration(127.0.0.1:33941, datanodeUuid=d82ec060-f13b-41ba-88a0-0486bb38a0a9, infoPort=38427, infoSecurePort=0, ipcPort=45839, storageInfo=lv=-57;cid=testClusterID;nsid=1340258790;c=1733462883129), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-06T05:28:05,649 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x270de150b695e1d1 with lease ID 0x315ab9ea1bed8188: Processing first storage report for DS-7753e9e5-6871-444b-9b6a-d2e2a3dbb8ad from datanode DatanodeRegistration(127.0.0.1:33941, datanodeUuid=d82ec060-f13b-41ba-88a0-0486bb38a0a9, infoPort=38427, infoSecurePort=0, ipcPort=45839, storageInfo=lv=-57;cid=testClusterID;nsid=1340258790;c=1733462883129)
2024-12-06T05:28:05,649 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x270de150b695e1d1 with lease ID 0x315ab9ea1bed8188: from storage DS-7753e9e5-6871-444b-9b6a-d2e2a3dbb8ad node DatanodeRegistration(127.0.0.1:33941, datanodeUuid=d82ec060-f13b-41ba-88a0-0486bb38a0a9, infoPort=38427, infoSecurePort=0, ipcPort=45839, storageInfo=lv=-57;cid=testClusterID;nsid=1340258790;c=1733462883129), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:28:05,665 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57
2024-12-06T05:28:05,669 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/cluster_4b9a14b3-6a40-37da-1c8a-68406eef4d27/zookeeper_0, clientPort=59149, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/cluster_4b9a14b3-6a40-37da-1c8a-68406eef4d27/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/cluster_4b9a14b3-6a40-37da-1c8a-68406eef4d27/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-06T05:28:05,670 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59149
2024-12-06T05:28:05,670 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:05,672 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:05,684 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741825_1001 (size=7)
2024-12-06T05:28:05,684 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41205 is added to blk_1073741825_1001 (size=7)
2024-12-06T05:28:05,686 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009 with version=8
2024-12-06T05:28:05,686 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/hbase-staging
2024-12-06T05:28:05,689 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/ec46afd49254:0 server-side Connection retries=45
2024-12-06T05:28:05,689 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:28:05,690 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T05:28:05,690 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T05:28:05,690 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:28:05,690 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T05:28:05,690 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-06T05:28:05,690 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T05:28:05,691 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36099
2024-12-06T05:28:05,693 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36099 connecting to ZooKeeper ensemble=127.0.0.1:59149
2024-12-06T05:28:05,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:360990x0, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T05:28:05,760 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36099-0x101a91c80cf0000 connected
2024-12-06T05:28:05,846 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:05,849 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:05,854 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:28:05,854 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009, hbase.cluster.distributed=false
2024-12-06T05:28:05,856 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T05:28:05,857 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36099
2024-12-06T05:28:05,857 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36099
2024-12-06T05:28:05,858 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36099
2024-12-06T05:28:05,858 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36099
2024-12-06T05:28:05,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36099
2024-12-06T05:28:05,874 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/ec46afd49254:0 server-side Connection retries=45
2024-12-06T05:28:05,874 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:28:05,874 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T05:28:05,874 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T05:28:05,874 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:28:05,874 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T05:28:05,874 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-06T05:28:05,875 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T05:28:05,875 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43675
2024-12-06T05:28:05,877 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43675 connecting to ZooKeeper ensemble=127.0.0.1:59149
2024-12-06T05:28:05,878 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:05,881 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:05,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:436750x0, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T05:28:05,897 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:436750x0, quorum=127.0.0.1:59149, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:28:05,897 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43675-0x101a91c80cf0001 connected
2024-12-06T05:28:05,897 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-06T05:28:05,899 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-06T05:28:05,899 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-06T05:28:05,900 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T05:28:05,903 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43675
2024-12-06T05:28:05,904 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43675
2024-12-06T05:28:05,904 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43675
2024-12-06T05:28:05,905 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43675
2024-12-06T05:28:05,905 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43675
2024-12-06T05:28:05,916 DEBUG [M:0;ec46afd49254:36099 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ec46afd49254:36099
2024-12-06T05:28:05,917 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ec46afd49254,36099,1733462885689
2024-12-06T05:28:05,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:28:05,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:28:05,931 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ec46afd49254,36099,1733462885689
2024-12-06T05:28:05,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:05,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-06T05:28:05,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:05,942 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-06T05:28:05,942 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ec46afd49254,36099,1733462885689 from backup master directory
2024-12-06T05:28:05,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ec46afd49254,36099,1733462885689
2024-12-06T05:28:05,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:28:05,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:28:05,951 WARN  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T05:28:05,951 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ec46afd49254,36099,1733462885689
2024-12-06T05:28:05,956 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/hbase.id] with ID: 991dc62d-1443-4245-b2ba-5afa08a244fb
2024-12-06T05:28:05,956 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/.tmp/hbase.id
2024-12-06T05:28:05,963 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741826_1002 (size=42)
2024-12-06T05:28:05,964 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41205 is added to blk_1073741826_1002 (size=42)
2024-12-06T05:28:05,964 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/.tmp/hbase.id]:[hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/hbase.id]
2024-12-06T05:28:05,978 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:05,978 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-06T05:28:05,980 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-12-06T05:28:05,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:05,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:06,006 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41205 is added to blk_1073741827_1003 (size=196)
2024-12-06T05:28:06,007 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741827_1003 (size=196)
2024-12-06T05:28:06,007 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T05:28:06,009 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-06T05:28:06,009 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:28:06,021 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741828_1004 (size=1189)
2024-12-06T05:28:06,021 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41205 is added to blk_1073741828_1004 (size=1189)
2024-12-06T05:28:06,022 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store
2024-12-06T05:28:06,029 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41205 is added to blk_1073741829_1005 (size=34)
2024-12-06T05:28:06,029 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741829_1005 (size=34)
2024-12-06T05:28:06,030 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:28:06,030 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T05:28:06,030 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:06,030 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:06,030 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T05:28:06,030 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:06,030 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:06,031 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733462886030Disabling compacts and flushes for region at 1733462886030Disabling writes for close at 1733462886030Writing region close event to WAL at 1733462886030Closed at 1733462886030
2024-12-06T05:28:06,032 WARN  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/.initializing
2024-12-06T05:28:06,032 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/WALs/ec46afd49254,36099,1733462885689
2024-12-06T05:28:06,035 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C36099%2C1733462885689, suffix=, logDir=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/WALs/ec46afd49254,36099,1733462885689, archiveDir=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/oldWALs, maxLogs=10
2024-12-06T05:28:06,035 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C36099%2C1733462885689.1733462886035
2024-12-06T05:28:06,041 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/WALs/ec46afd49254,36099,1733462885689/ec46afd49254%2C36099%2C1733462885689.1733462886035
2024-12-06T05:28:06,044 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39751:39751),(127.0.0.1/127.0.0.1:38427:38427)]
2024-12-06T05:28:06,045 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:28:06,045 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:28:06,045 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:28:06,045 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:28:06,047 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:28:06,048 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-06T05:28:06,049 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:06,049 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:06,049 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:28:06,051 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-06T05:28:06,051 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:06,051 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:28:06,051 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:28:06,053 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-06T05:28:06,054 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:06,054 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:28:06,055 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:28:06,056 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-06T05:28:06,056 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:06,057 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:28:06,057 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:28:06,058 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:28:06,058 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:28:06,060 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:28:06,060 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:28:06,061 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-06T05:28:06,062 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:28:06,064 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:28:06,065 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=728944, jitterRate=-0.07310087978839874}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-06T05:28:06,066 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733462886046Initializing all the Stores at 1733462886047 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462886047Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462886047Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462886047Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462886047Cleaning up temporary data from old regions at 1733462886060 (+13 ms)Region opened successfully at 1733462886066 (+6 ms)
2024-12-06T05:28:06,066 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-06T05:28:06,072 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6466e6d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0
2024-12-06T05:28:06,073 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-06T05:28:06,073 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-06T05:28:06,073 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-06T05:28:06,074 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-06T05:28:06,075 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-06T05:28:06,075 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-06T05:28:06,075 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-06T05:28:06,078 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-06T05:28:06,079 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-06T05:28:06,088 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-06T05:28:06,088 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-06T05:28:06,089 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-06T05:28:06,098 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-06T05:28:06,099 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-06T05:28:06,100 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-06T05:28:06,109 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-06T05:28:06,110 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-06T05:28:06,119 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-06T05:28:06,123 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-06T05:28:06,130 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-06T05:28:06,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T05:28:06,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T05:28:06,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:06,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:06,142 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ec46afd49254,36099,1733462885689, sessionid=0x101a91c80cf0000, setting cluster-up flag (Was=false)
2024-12-06T05:28:06,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:06,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:06,194 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-06T05:28:06,197 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec46afd49254,36099,1733462885689
2024-12-06T05:28:06,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:06,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:06,246 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-06T05:28:06,249 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec46afd49254,36099,1733462885689
2024-12-06T05:28:06,251 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-06T05:28:06,255 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-06T05:28:06,255 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-06T05:28:06,256 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-06T05:28:06,256 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ec46afd49254,36099,1733462885689
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-06T05:28:06,260 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:28:06,260 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:28:06,260 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:28:06,260 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:28:06,260 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ec46afd49254:0, corePoolSize=10, maxPoolSize=10
2024-12-06T05:28:06,260 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:06,261 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ec46afd49254:0, corePoolSize=2, maxPoolSize=2
2024-12-06T05:28:06,261 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:06,262 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733462916261
2024-12-06T05:28:06,262 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-06T05:28:06,262 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-06T05:28:06,262 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-06T05:28:06,262 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-06T05:28:06,262 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-06T05:28:06,262 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-06T05:28:06,262 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:06,263 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-06T05:28:06,263 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:28:06,263 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-06T05:28:06,263 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-06T05:28:06,263 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-06T05:28:06,263 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-06T05:28:06,264 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-06T05:28:06,264 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462886264,5,FailOnTimeoutGroup]
2024-12-06T05:28:06,264 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462886264,5,FailOnTimeoutGroup]
2024-12-06T05:28:06,264 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:06,264 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-06T05:28:06,264 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:06,264 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:06,265 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:06,265 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-06T05:28:06,273 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41205 is added to blk_1073741831_1007 (size=1321)
2024-12-06T05:28:06,273 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741831_1007 (size=1321)
2024-12-06T05:28:06,275 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-06T05:28:06,275 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009
2024-12-06T05:28:06,287 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41205 is added to blk_1073741832_1008 (size=32)
2024-12-06T05:28:06,287 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741832_1008 (size=32)
2024-12-06T05:28:06,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:28:06,290 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T05:28:06,292 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T05:28:06,292 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:06,293 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:06,293 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T05:28:06,295 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T05:28:06,295 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:06,296 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:06,296 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T05:28:06,298 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T05:28:06,298 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:06,299 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:06,299 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T05:28:06,301 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T05:28:06,301 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:06,301 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:06,302 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T05:28:06,302 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/data/hbase/meta/1588230740
2024-12-06T05:28:06,303 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/data/hbase/meta/1588230740
2024-12-06T05:28:06,305 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T05:28:06,305 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T05:28:06,305 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T05:28:06,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T05:28:06,307 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.HRegionServer(746): ClusterId : 991dc62d-1443-4245-b2ba-5afa08a244fb
2024-12-06T05:28:06,307 DEBUG [RS:0;ec46afd49254:43675 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-06T05:28:06,309 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:28:06,309 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=720558, jitterRate=-0.08376315236091614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T05:28:06,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733462886288Initializing all the Stores at 1733462886290 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462886290Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462886290Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462886290Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462886290Cleaning up temporary data from old regions at 1733462886305 (+15 ms)Region opened successfully at 1733462886311 (+6 ms)
2024-12-06T05:28:06,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T05:28:06,311 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T05:28:06,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T05:28:06,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T05:28:06,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T05:28:06,311 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T05:28:06,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733462886311Disabling compacts and flushes for region at 1733462886311Disabling writes for close at 1733462886311Writing region close event to WAL at 1733462886311Closed at 1733462886311
2024-12-06T05:28:06,313 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:28:06,313 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-06T05:28:06,313 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-06T05:28:06,315 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T05:28:06,316 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-06T05:28:06,320 DEBUG [RS:0;ec46afd49254:43675 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-06T05:28:06,321 DEBUG [RS:0;ec46afd49254:43675 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-06T05:28:06,331 DEBUG [RS:0;ec46afd49254:43675 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-06T05:28:06,332 DEBUG [RS:0;ec46afd49254:43675 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16afaf9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0
2024-12-06T05:28:06,343 DEBUG [RS:0;ec46afd49254:43675 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ec46afd49254:43675
2024-12-06T05:28:06,343 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-06T05:28:06,343 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-06T05:28:06,343 DEBUG [RS:0;ec46afd49254:43675 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-06T05:28:06,344 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec46afd49254,36099,1733462885689 with port=43675, startcode=1733462885874
2024-12-06T05:28:06,345 DEBUG [RS:0;ec46afd49254:43675 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-06T05:28:06,347 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59033, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-12-06T05:28:06,348 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36099 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ec46afd49254,43675,1733462885874
2024-12-06T05:28:06,348 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36099 {}] master.ServerManager(517): Registering regionserver=ec46afd49254,43675,1733462885874
2024-12-06T05:28:06,350 DEBUG [RS:0;ec46afd49254:43675 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009
2024-12-06T05:28:06,350 DEBUG [RS:0;ec46afd49254:43675 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35237
2024-12-06T05:28:06,350 DEBUG [RS:0;ec46afd49254:43675 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-06T05:28:06,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T05:28:06,362 DEBUG [RS:0;ec46afd49254:43675 {}] zookeeper.ZKUtil(111): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec46afd49254,43675,1733462885874
2024-12-06T05:28:06,362 WARN  [RS:0;ec46afd49254:43675 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T05:28:06,363 INFO  [RS:0;ec46afd49254:43675 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:28:06,363 DEBUG [RS:0;ec46afd49254:43675 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/WALs/ec46afd49254,43675,1733462885874
2024-12-06T05:28:06,363 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec46afd49254,43675,1733462885874]
2024-12-06T05:28:06,367 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-06T05:28:06,369 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-06T05:28:06,370 INFO  [RS:0;ec46afd49254:43675 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T05:28:06,370 INFO  [RS:0;ec46afd49254:43675 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:06,370 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-06T05:28:06,371 INFO  [RS:0;ec46afd49254:43675 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-06T05:28:06,371 INFO  [RS:0;ec46afd49254:43675 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:06,371 DEBUG [RS:0;ec46afd49254:43675 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:06,371 DEBUG [RS:0;ec46afd49254:43675 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:06,372 DEBUG [RS:0;ec46afd49254:43675 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:06,372 DEBUG [RS:0;ec46afd49254:43675 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:06,372 DEBUG [RS:0;ec46afd49254:43675 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:06,372 DEBUG [RS:0;ec46afd49254:43675 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0, corePoolSize=2, maxPoolSize=2
2024-12-06T05:28:06,372 DEBUG [RS:0;ec46afd49254:43675 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:06,372 DEBUG [RS:0;ec46afd49254:43675 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:06,372 DEBUG [RS:0;ec46afd49254:43675 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:06,372 DEBUG [RS:0;ec46afd49254:43675 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:06,372 DEBUG [RS:0;ec46afd49254:43675 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:06,372 DEBUG [RS:0;ec46afd49254:43675 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:06,372 DEBUG [RS:0;ec46afd49254:43675 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3
2024-12-06T05:28:06,372 DEBUG [RS:0;ec46afd49254:43675 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3
2024-12-06T05:28:06,374 INFO  [RS:0;ec46afd49254:43675 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:06,374 INFO  [RS:0;ec46afd49254:43675 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:06,374 INFO  [RS:0;ec46afd49254:43675 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:06,374 INFO  [RS:0;ec46afd49254:43675 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:06,374 INFO  [RS:0;ec46afd49254:43675 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:06,374 INFO  [RS:0;ec46afd49254:43675 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43675,1733462885874-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T05:28:06,388 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-06T05:28:06,388 INFO  [RS:0;ec46afd49254:43675 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43675,1733462885874-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:06,388 INFO  [RS:0;ec46afd49254:43675 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:06,388 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.Replication(171): ec46afd49254,43675,1733462885874 started
2024-12-06T05:28:06,402 INFO  [RS:0;ec46afd49254:43675 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:06,402 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.HRegionServer(1482): Serving as ec46afd49254,43675,1733462885874, RpcServer on ec46afd49254/172.17.0.3:43675, sessionid=0x101a91c80cf0001
2024-12-06T05:28:06,402 DEBUG [RS:0;ec46afd49254:43675 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-06T05:28:06,402 DEBUG [RS:0;ec46afd49254:43675 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec46afd49254,43675,1733462885874
2024-12-06T05:28:06,402 DEBUG [RS:0;ec46afd49254:43675 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,43675,1733462885874'
2024-12-06T05:28:06,402 DEBUG [RS:0;ec46afd49254:43675 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-06T05:28:06,403 DEBUG [RS:0;ec46afd49254:43675 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-06T05:28:06,404 DEBUG [RS:0;ec46afd49254:43675 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-06T05:28:06,404 DEBUG [RS:0;ec46afd49254:43675 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-06T05:28:06,404 DEBUG [RS:0;ec46afd49254:43675 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec46afd49254,43675,1733462885874
2024-12-06T05:28:06,404 DEBUG [RS:0;ec46afd49254:43675 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,43675,1733462885874'
2024-12-06T05:28:06,404 DEBUG [RS:0;ec46afd49254:43675 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-06T05:28:06,404 DEBUG [RS:0;ec46afd49254:43675 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-06T05:28:06,405 DEBUG [RS:0;ec46afd49254:43675 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-06T05:28:06,405 INFO  [RS:0;ec46afd49254:43675 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-06T05:28:06,405 INFO  [RS:0;ec46afd49254:43675 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-06T05:28:06,467 WARN  [ec46afd49254:36099 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-06T05:28:06,506 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T05:28:06,506 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-06T05:28:06,508 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-06T05:28:06,510 INFO  [RS:0;ec46afd49254:43675 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C43675%2C1733462885874, suffix=, logDir=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/WALs/ec46afd49254,43675,1733462885874, archiveDir=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/oldWALs, maxLogs=32
2024-12-06T05:28:06,513 INFO  [RS:0;ec46afd49254:43675 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C43675%2C1733462885874.1733462886513
2024-12-06T05:28:06,522 INFO  [RS:0;ec46afd49254:43675 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/WALs/ec46afd49254,43675,1733462885874/ec46afd49254%2C43675%2C1733462885874.1733462886513
2024-12-06T05:28:06,524 DEBUG [RS:0;ec46afd49254:43675 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39751:39751),(127.0.0.1/127.0.0.1:38427:38427)]
2024-12-06T05:28:06,717 DEBUG [ec46afd49254:36099 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-06T05:28:06,719 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ec46afd49254,43675,1733462885874
2024-12-06T05:28:06,723 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec46afd49254,43675,1733462885874, state=OPENING
2024-12-06T05:28:06,741 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-06T05:28:06,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:06,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:06,753 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T05:28:06,753 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:28:06,753 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:28:06,753 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ec46afd49254,43675,1733462885874}]
2024-12-06T05:28:06,910 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-06T05:28:06,912 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36407, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-06T05:28:06,917 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-06T05:28:06,917 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:28:06,920 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C43675%2C1733462885874.meta, suffix=.meta, logDir=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/WALs/ec46afd49254,43675,1733462885874, archiveDir=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/oldWALs, maxLogs=32
2024-12-06T05:28:06,922 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C43675%2C1733462885874.meta.1733462886922.meta
2024-12-06T05:28:06,929 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/WALs/ec46afd49254,43675,1733462885874/ec46afd49254%2C43675%2C1733462885874.meta.1733462886922.meta
2024-12-06T05:28:06,930 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38427:38427),(127.0.0.1/127.0.0.1:39751:39751)]
2024-12-06T05:28:06,931 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:28:06,931 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-06T05:28:06,932 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-06T05:28:06,932 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-06T05:28:06,932 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-06T05:28:06,932 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:28:06,932 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-06T05:28:06,932 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-06T05:28:06,934 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T05:28:06,935 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T05:28:06,935 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:06,936 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:06,936 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T05:28:06,937 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T05:28:06,937 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:06,937 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:06,937 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T05:28:06,938 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T05:28:06,939 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:06,939 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:06,939 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T05:28:06,941 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T05:28:06,941 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:06,941 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:06,942 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T05:28:06,943 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/data/hbase/meta/1588230740
2024-12-06T05:28:06,944 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/data/hbase/meta/1588230740
2024-12-06T05:28:06,946 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T05:28:06,946 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T05:28:06,946 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T05:28:06,948 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T05:28:06,949 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=718325, jitterRate=-0.086602583527565}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T05:28:06,949 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-06T05:28:06,950 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733462886932Writing region info on filesystem at 1733462886932Initializing all the Stores at 1733462886934 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462886934Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462886934Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462886934Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462886934Cleaning up temporary data from old regions at 1733462886946 (+12 ms)Running coprocessor post-open hooks at 1733462886949 (+3 ms)Region opened successfully at 1733462886950 (+1 ms)
2024-12-06T05:28:06,951 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733462886909
2024-12-06T05:28:06,954 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-06T05:28:06,954 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-06T05:28:06,955 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ec46afd49254,43675,1733462885874
2024-12-06T05:28:06,956 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec46afd49254,43675,1733462885874, state=OPEN
2024-12-06T05:28:06,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T05:28:06,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T05:28:06,995 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ec46afd49254,43675,1733462885874
2024-12-06T05:28:06,995 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:28:06,995 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:28:07,001 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-06T05:28:07,001 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec46afd49254,43675,1733462885874 in 242 msec
2024-12-06T05:28:07,006 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-06T05:28:07,006 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 688 msec
2024-12-06T05:28:07,007 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:28:07,008 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-06T05:28:07,010 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T05:28:07,010 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec46afd49254,43675,1733462885874, seqNum=-1]
2024-12-06T05:28:07,011 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T05:28:07,013 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56257, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T05:28:07,021 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 766 msec
2024-12-06T05:28:07,021 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733462887021, completionTime=-1
2024-12-06T05:28:07,021 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-06T05:28:07,021 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-06T05:28:07,024 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-06T05:28:07,024 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733462947024
2024-12-06T05:28:07,024 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733463007024
2024-12-06T05:28:07,024 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-06T05:28:07,024 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,36099,1733462885689-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:07,024 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,36099,1733462885689-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:07,024 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,36099,1733462885689-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:07,025 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ec46afd49254:36099, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:07,025 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:07,025 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:07,027 DEBUG [master/ec46afd49254:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T05:28:07,030 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.078sec
2024-12-06T05:28:07,030 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-06T05:28:07,030 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-06T05:28:07,030 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-06T05:28:07,030 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-06T05:28:07,030 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-06T05:28:07,030 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,36099,1733462885689-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T05:28:07,030 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,36099,1733462885689-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-06T05:28:07,033 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-06T05:28:07,034 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-06T05:28:07,034 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,36099,1733462885689-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:07,108 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57927b81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:28:07,109 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ec46afd49254,36099,-1 for getting cluster id
2024-12-06T05:28:07,109 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-06T05:28:07,113 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '991dc62d-1443-4245-b2ba-5afa08a244fb'
2024-12-06T05:28:07,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-06T05:28:07,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "991dc62d-1443-4245-b2ba-5afa08a244fb"

2024-12-06T05:28:07,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d34ab95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:28:07,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ec46afd49254,36099,-1]
2024-12-06T05:28:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-06T05:28:07,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:07,119 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-06T05:28:07,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aa3a1e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:28:07,120 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T05:28:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec46afd49254,43675,1733462885874, seqNum=-1]
2024-12-06T05:28:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T05:28:07,123 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T05:28:07,125 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ec46afd49254,36099,1733462885689
2024-12-06T05:28:07,125 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:07,128 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-06T05:28:07,128 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-06T05:28:07,128 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T05:28:07,129 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:28:07,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:07,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:07,129 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-06T05:28:07,129 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-06T05:28:07,129 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=310993063, stopped=false
2024-12-06T05:28:07,129 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ec46afd49254,36099,1733462885689
2024-12-06T05:28:07,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T05:28:07,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T05:28:07,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:07,149 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T05:28:07,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:07,149 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T05:28:07,149 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:28:07,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:07,150 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:28:07,150 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:28:07,150 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ec46afd49254,43675,1733462885874' *****
2024-12-06T05:28:07,150 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-06T05:28:07,150 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-06T05:28:07,150 INFO  [RS:0;ec46afd49254:43675 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-06T05:28:07,150 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-06T05:28:07,150 INFO  [RS:0;ec46afd49254:43675 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-06T05:28:07,150 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.HRegionServer(959): stopping server ec46afd49254,43675,1733462885874
2024-12-06T05:28:07,151 INFO  [RS:0;ec46afd49254:43675 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T05:28:07,151 INFO  [RS:0;ec46afd49254:43675 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ec46afd49254:43675.
2024-12-06T05:28:07,151 DEBUG [RS:0;ec46afd49254:43675 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:28:07,151 DEBUG [RS:0;ec46afd49254:43675 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:07,151 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-06T05:28:07,151 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-06T05:28:07,151 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-06T05:28:07,151 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-06T05:28:07,151 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-06T05:28:07,151 DEBUG [RS:0;ec46afd49254:43675 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-06T05:28:07,151 DEBUG [RS:0;ec46afd49254:43675 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-06T05:28:07,151 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T05:28:07,151 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T05:28:07,152 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T05:28:07,152 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T05:28:07,152 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T05:28:07,152 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-12-06T05:28:07,167 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/data/hbase/meta/1588230740/.tmp/ns/1e54d5ff70f7448bac0325b5b7dac759 is 43, key is default/ns:d/1733462887014/Put/seqid=0
2024-12-06T05:28:07,173 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41205 is added to blk_1073741835_1011 (size=5153)
2024-12-06T05:28:07,174 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741835_1011 (size=5153)
2024-12-06T05:28:07,174 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/data/hbase/meta/1588230740/.tmp/ns/1e54d5ff70f7448bac0325b5b7dac759
2024-12-06T05:28:07,183 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/data/hbase/meta/1588230740/.tmp/ns/1e54d5ff70f7448bac0325b5b7dac759 as hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/data/hbase/meta/1588230740/ns/1e54d5ff70f7448bac0325b5b7dac759
2024-12-06T05:28:07,192 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/data/hbase/meta/1588230740/ns/1e54d5ff70f7448bac0325b5b7dac759, entries=2, sequenceid=6, filesize=5.0 K
2024-12-06T05:28:07,194 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 42ms, sequenceid=6, compaction requested=false
2024-12-06T05:28:07,194 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-06T05:28:07,200 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-06T05:28:07,201 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T05:28:07,201 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T05:28:07,201 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733462887151Running coprocessor pre-close hooks at 1733462887151Disabling compacts and flushes for region at 1733462887151Disabling writes for close at 1733462887152 (+1 ms)Obtaining lock to block concurrent updates at 1733462887152Preparing flush snapshotting stores in 1588230740 at 1733462887152Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733462887152Flushing stores of hbase:meta,,1.1588230740 at 1733462887153 (+1 ms)Flushing 1588230740/ns: creating writer at 1733462887153Flushing 1588230740/ns: appending metadata at 1733462887167 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733462887167Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2de928c2: reopening flushed file at 1733462887182 (+15 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 42ms, sequenceid=6, compaction requested=false at 1733462887194 (+12 ms)Writing region close event to WAL at 1733462887196 (+2 ms)Running coprocessor post-close hooks at 1733462887201 (+5 ms)Closed at 1733462887201
2024-12-06T05:28:07,201 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-06T05:28:07,352 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.HRegionServer(976): stopping server ec46afd49254,43675,1733462885874; all regions closed.
2024-12-06T05:28:07,352 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:07,352 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:07,353 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:07,353 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:07,353 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:07,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41205 is added to blk_1073741834_1010 (size=1152)
2024-12-06T05:28:07,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741834_1010 (size=1152)
2024-12-06T05:28:07,358 DEBUG [RS:0;ec46afd49254:43675 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/oldWALs
2024-12-06T05:28:07,358 INFO  [RS:0;ec46afd49254:43675 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ec46afd49254%2C43675%2C1733462885874.meta:.meta(num 1733462886922)
2024-12-06T05:28:07,359 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:07,359 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:07,359 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:07,359 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:07,359 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:07,362 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741833_1009 (size=93)
2024-12-06T05:28:07,362 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41205 is added to blk_1073741833_1009 (size=93)
2024-12-06T05:28:07,364 DEBUG [RS:0;ec46afd49254:43675 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/oldWALs
2024-12-06T05:28:07,364 INFO  [RS:0;ec46afd49254:43675 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ec46afd49254%2C43675%2C1733462885874:(num 1733462886513)
2024-12-06T05:28:07,364 DEBUG [RS:0;ec46afd49254:43675 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:07,364 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T05:28:07,364 INFO  [RS:0;ec46afd49254:43675 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T05:28:07,364 INFO  [RS:0;ec46afd49254:43675 {}] hbase.ChoreService(370): Chore service for: regionserver/ec46afd49254:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-06T05:28:07,364 INFO  [RS:0;ec46afd49254:43675 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T05:28:07,364 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T05:28:07,364 INFO  [RS:0;ec46afd49254:43675 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43675
2024-12-06T05:28:07,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T05:28:07,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec46afd49254,43675,1733462885874
2024-12-06T05:28:07,383 INFO  [RS:0;ec46afd49254:43675 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T05:28:07,404 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec46afd49254,43675,1733462885874]
2024-12-06T05:28:07,414 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ec46afd49254,43675,1733462885874 already deleted, retry=false
2024-12-06T05:28:07,414 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ec46afd49254,43675,1733462885874 expired; onlineServers=0
2024-12-06T05:28:07,415 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ec46afd49254,36099,1733462885689' *****
2024-12-06T05:28:07,415 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-06T05:28:07,415 INFO  [M:0;ec46afd49254:36099 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T05:28:07,415 INFO  [M:0;ec46afd49254:36099 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T05:28:07,415 DEBUG [M:0;ec46afd49254:36099 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-06T05:28:07,416 DEBUG [M:0;ec46afd49254:36099 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-06T05:28:07,416 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-06T05:28:07,416 DEBUG [master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462886264 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462886264,5,FailOnTimeoutGroup]
2024-12-06T05:28:07,416 DEBUG [master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462886264 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462886264,5,FailOnTimeoutGroup]
2024-12-06T05:28:07,416 INFO  [M:0;ec46afd49254:36099 {}] hbase.ChoreService(370): Chore service for: master/ec46afd49254:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-06T05:28:07,416 INFO  [M:0;ec46afd49254:36099 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T05:28:07,417 DEBUG [M:0;ec46afd49254:36099 {}] master.HMaster(1795): Stopping service threads
2024-12-06T05:28:07,417 INFO  [M:0;ec46afd49254:36099 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-06T05:28:07,417 INFO  [M:0;ec46afd49254:36099 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T05:28:07,417 INFO  [M:0;ec46afd49254:36099 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-06T05:28:07,418 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-06T05:28:07,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-06T05:28:07,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:07,425 DEBUG [M:0;ec46afd49254:36099 {}] zookeeper.ZKUtil(347): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-06T05:28:07,425 WARN  [M:0;ec46afd49254:36099 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-06T05:28:07,426 INFO  [M:0;ec46afd49254:36099 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/.lastflushedseqids
2024-12-06T05:28:07,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41205 is added to blk_1073741836_1012 (size=99)
2024-12-06T05:28:07,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741836_1012 (size=99)
2024-12-06T05:28:07,432 INFO  [M:0;ec46afd49254:36099 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-06T05:28:07,432 INFO  [M:0;ec46afd49254:36099 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-06T05:28:07,432 DEBUG [M:0;ec46afd49254:36099 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T05:28:07,432 INFO  [M:0;ec46afd49254:36099 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:07,432 DEBUG [M:0;ec46afd49254:36099 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:07,432 DEBUG [M:0;ec46afd49254:36099 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T05:28:07,432 DEBUG [M:0;ec46afd49254:36099 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:07,432 INFO  [M:0;ec46afd49254:36099 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-12-06T05:28:07,451 DEBUG [M:0;ec46afd49254:36099 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/edf4be6b5914448691db8fc2b436347d is 82, key is hbase:meta,,1/info:regioninfo/1733462886955/Put/seqid=0
2024-12-06T05:28:07,457 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741837_1013 (size=5672)
2024-12-06T05:28:07,457 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41205 is added to blk_1073741837_1013 (size=5672)
2024-12-06T05:28:07,458 INFO  [M:0;ec46afd49254:36099 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/edf4be6b5914448691db8fc2b436347d
2024-12-06T05:28:07,472 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:07,477 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:07,482 DEBUG [M:0;ec46afd49254:36099 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a3bf74f596924540936102fe8a9abf8a is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733462887020/Put/seqid=0
2024-12-06T05:28:07,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741838_1014 (size=5275)
2024-12-06T05:28:07,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41205 is added to blk_1073741838_1014 (size=5275)
2024-12-06T05:28:07,489 INFO  [M:0;ec46afd49254:36099 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a3bf74f596924540936102fe8a9abf8a
2024-12-06T05:28:07,504 INFO  [RS:0;ec46afd49254:43675 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T05:28:07,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:28:07,504 INFO  [RS:0;ec46afd49254:43675 {}] regionserver.HRegionServer(1031): Exiting; stopping=ec46afd49254,43675,1733462885874; zookeeper connection closed.
2024-12-06T05:28:07,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43675-0x101a91c80cf0001, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:28:07,504 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2975e0f5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2975e0f5
2024-12-06T05:28:07,504 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-06T05:28:07,508 DEBUG [M:0;ec46afd49254:36099 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97a3a42265fa4f43bcf8048155b4f078 is 69, key is ec46afd49254,43675,1733462885874/rs:state/1733462886348/Put/seqid=0
2024-12-06T05:28:07,512 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41205 is added to blk_1073741839_1015 (size=5156)
2024-12-06T05:28:07,512 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741839_1015 (size=5156)
2024-12-06T05:28:07,513 INFO  [M:0;ec46afd49254:36099 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97a3a42265fa4f43bcf8048155b4f078
2024-12-06T05:28:07,531 DEBUG [M:0;ec46afd49254:36099 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ae4060e94fae4d328345edeac0d57867 is 52, key is load_balancer_on/state:d/1733462887127/Put/seqid=0
2024-12-06T05:28:07,536 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741840_1016 (size=5056)
2024-12-06T05:28:07,537 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41205 is added to blk_1073741840_1016 (size=5056)
2024-12-06T05:28:07,537 INFO  [M:0;ec46afd49254:36099 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ae4060e94fae4d328345edeac0d57867
2024-12-06T05:28:07,543 DEBUG [M:0;ec46afd49254:36099 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/edf4be6b5914448691db8fc2b436347d as hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/edf4be6b5914448691db8fc2b436347d
2024-12-06T05:28:07,550 INFO  [M:0;ec46afd49254:36099 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/edf4be6b5914448691db8fc2b436347d, entries=8, sequenceid=29, filesize=5.5 K
2024-12-06T05:28:07,551 DEBUG [M:0;ec46afd49254:36099 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a3bf74f596924540936102fe8a9abf8a as hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a3bf74f596924540936102fe8a9abf8a
2024-12-06T05:28:07,559 INFO  [M:0;ec46afd49254:36099 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a3bf74f596924540936102fe8a9abf8a, entries=3, sequenceid=29, filesize=5.2 K
2024-12-06T05:28:07,560 DEBUG [M:0;ec46afd49254:36099 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97a3a42265fa4f43bcf8048155b4f078 as hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/97a3a42265fa4f43bcf8048155b4f078
2024-12-06T05:28:07,567 INFO  [M:0;ec46afd49254:36099 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/97a3a42265fa4f43bcf8048155b4f078, entries=1, sequenceid=29, filesize=5.0 K
2024-12-06T05:28:07,569 DEBUG [M:0;ec46afd49254:36099 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ae4060e94fae4d328345edeac0d57867 as hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ae4060e94fae4d328345edeac0d57867
2024-12-06T05:28:07,575 INFO  [M:0;ec46afd49254:36099 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35237/user/jenkins/test-data/e19947e1-244f-3626-c55f-2dd629d86009/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ae4060e94fae4d328345edeac0d57867, entries=1, sequenceid=29, filesize=4.9 K
2024-12-06T05:28:07,576 INFO  [M:0;ec46afd49254:36099 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 144ms, sequenceid=29, compaction requested=false
2024-12-06T05:28:07,578 INFO  [M:0;ec46afd49254:36099 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:07,578 DEBUG [M:0;ec46afd49254:36099 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733462887432Disabling compacts and flushes for region at 1733462887432Disabling writes for close at 1733462887432Obtaining lock to block concurrent updates at 1733462887432Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733462887432Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733462887433 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733462887434 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733462887434Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733462887451 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733462887451Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733462887464 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733462887481 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733462887482 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733462887494 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733462887507 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733462887507Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733462887518 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733462887531 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733462887531Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18046427: reopening flushed file at 1733462887542 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14e16621: reopening flushed file at 1733462887550 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23814603: reopening flushed file at 1733462887559 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55e49704: reopening flushed file at 1733462887567 (+8 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 144ms, sequenceid=29, compaction requested=false at 1733462887576 (+9 ms)Writing region close event to WAL at 1733462887578 (+2 ms)Closed at 1733462887578
2024-12-06T05:28:07,579 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:07,579 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:07,579 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:07,580 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:07,580 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:07,582 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741830_1006 (size=10311)
2024-12-06T05:28:07,582 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41205 is added to blk_1073741830_1006 (size=10311)
2024-12-06T05:28:07,583 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T05:28:07,583 INFO  [M:0;ec46afd49254:36099 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-06T05:28:07,583 INFO  [M:0;ec46afd49254:36099 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36099
2024-12-06T05:28:07,583 INFO  [M:0;ec46afd49254:36099 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T05:28:07,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:28:07,693 INFO  [M:0;ec46afd49254:36099 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T05:28:07,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36099-0x101a91c80cf0000, quorum=127.0.0.1:59149, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:28:07,699 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c17f9e5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:07,700 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@669fbf01{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:28:07,700 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:28:07,701 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d5e070a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:28:07,701 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a742c1f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/hadoop.log.dir/,STOPPED}
2024-12-06T05:28:07,703 WARN  [BP-1692227801-172.17.0.3-1733462883129 heartbeating to localhost/127.0.0.1:35237 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:28:07,703 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:28:07,703 WARN  [BP-1692227801-172.17.0.3-1733462883129 heartbeating to localhost/127.0.0.1:35237 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1692227801-172.17.0.3-1733462883129 (Datanode Uuid d82ec060-f13b-41ba-88a0-0486bb38a0a9) service to localhost/127.0.0.1:35237
2024-12-06T05:28:07,703 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:28:07,704 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/cluster_4b9a14b3-6a40-37da-1c8a-68406eef4d27/data/data3/current/BP-1692227801-172.17.0.3-1733462883129 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:07,704 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/cluster_4b9a14b3-6a40-37da-1c8a-68406eef4d27/data/data4/current/BP-1692227801-172.17.0.3-1733462883129 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:07,705 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:28:07,707 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@66b9e907{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:07,707 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@51f8cb4f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:28:07,707 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:28:07,707 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1aa9c156{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:28:07,708 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b58749b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/hadoop.log.dir/,STOPPED}
2024-12-06T05:28:07,709 WARN  [BP-1692227801-172.17.0.3-1733462883129 heartbeating to localhost/127.0.0.1:35237 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:28:07,709 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:28:07,709 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:28:07,709 WARN  [BP-1692227801-172.17.0.3-1733462883129 heartbeating to localhost/127.0.0.1:35237 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1692227801-172.17.0.3-1733462883129 (Datanode Uuid f1551871-761a-491b-beca-c9ef99893f65) service to localhost/127.0.0.1:35237
2024-12-06T05:28:07,709 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/cluster_4b9a14b3-6a40-37da-1c8a-68406eef4d27/data/data1/current/BP-1692227801-172.17.0.3-1733462883129 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:07,710 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/cluster_4b9a14b3-6a40-37da-1c8a-68406eef4d27/data/data2/current/BP-1692227801-172.17.0.3-1733462883129 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:07,710 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:28:07,715 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2a7d11e5{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T05:28:07,715 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5d4bdc00{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:28:07,716 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:28:07,716 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@641eaf99{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:28:07,716 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@737d6115{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/hadoop.log.dir/,STOPPED}
2024-12-06T05:28:07,722 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-06T05:28:07,739 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-06T05:28:07,739 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-06T05:28:07,739 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/hadoop.log.dir so I do NOT create it in target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642
2024-12-06T05:28:07,739 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ab20cdb0-741d-947d-d209-701dfa734b57/hadoop.tmp.dir so I do NOT create it in target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642
2024-12-06T05:28:07,739 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145, deleteOnExit=true
2024-12-06T05:28:07,739 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-06T05:28:07,739 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/test.cache.data in system properties and HBase conf
2024-12-06T05:28:07,739 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.tmp.dir in system properties and HBase conf
2024-12-06T05:28:07,739 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.log.dir in system properties and HBase conf
2024-12-06T05:28:07,739 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-06T05:28:07,740 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-06T05:28:07,740 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-06T05:28:07,740 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-06T05:28:07,740 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-06T05:28:07,740 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-06T05:28:07,740 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-06T05:28:07,740 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T05:28:07,740 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-06T05:28:07,740 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-06T05:28:07,741 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T05:28:07,741 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T05:28:07,741 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-06T05:28:07,741 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/nfs.dump.dir in system properties and HBase conf
2024-12-06T05:28:07,741 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/java.io.tmpdir in system properties and HBase conf
2024-12-06T05:28:07,741 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T05:28:07,741 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-06T05:28:07,741 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-06T05:28:07,753 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T05:28:07,986 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T05:28:07,988 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:08,001 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:08,003 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:08,004 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:08,053 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:28:08,058 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:28:08,059 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:28:08,059 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:28:08,060 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T05:28:08,060 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:28:08,061 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1aa07d80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:28:08,061 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3150e6db{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:28:08,161 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2606b08f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/java.io.tmpdir/jetty-localhost-39279-hadoop-hdfs-3_4_1-tests_jar-_-any-14189522645176162199/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T05:28:08,161 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@c053989{HTTP/1.1, (http/1.1)}{localhost:39279}
2024-12-06T05:28:08,162 INFO  [Time-limited test {}] server.Server(415): Started @107773ms
2024-12-06T05:28:08,174 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T05:28:08,375 INFO  [regionserver/ec46afd49254:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T05:28:08,408 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:28:08,411 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:28:08,412 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:28:08,412 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:28:08,412 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T05:28:08,413 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@180ba686{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:28:08,413 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@21f2acf7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:28:08,505 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5afc739a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/java.io.tmpdir/jetty-localhost-38791-hadoop-hdfs-3_4_1-tests_jar-_-any-9497893360391737362/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:08,505 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@200d0f88{HTTP/1.1, (http/1.1)}{localhost:38791}
2024-12-06T05:28:08,505 INFO  [Time-limited test {}] server.Server(415): Started @108116ms
2024-12-06T05:28:08,506 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:28:08,533 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:28:08,536 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:28:08,537 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:28:08,537 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:28:08,537 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T05:28:08,537 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@57ebe64c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:28:08,538 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2dc8ddff{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:28:08,638 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@48b0be64{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/java.io.tmpdir/jetty-localhost-35531-hadoop-hdfs-3_4_1-tests_jar-_-any-2620906410335209060/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:08,638 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4cae84f2{HTTP/1.1, (http/1.1)}{localhost:35531}
2024-12-06T05:28:08,638 INFO  [Time-limited test {}] server.Server(415): Started @108249ms
2024-12-06T05:28:08,639 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:28:09,906 WARN  [Thread-673 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data1/current/BP-2129024424-172.17.0.3-1733462887764/current, will proceed with Du for space computation calculation, 
2024-12-06T05:28:09,906 WARN  [Thread-674 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data2/current/BP-2129024424-172.17.0.3-1733462887764/current, will proceed with Du for space computation calculation, 
2024-12-06T05:28:09,926 WARN  [Thread-637 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:28:09,928 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5964e7cc817c54e7 with lease ID 0xba9d9ca3b0b8becd: Processing first storage report for DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd from datanode DatanodeRegistration(127.0.0.1:41475, datanodeUuid=6e5f6fb1-d15a-4191-9fe6-f53a63d1617f, infoPort=35855, infoSecurePort=0, ipcPort=46841, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764)
2024-12-06T05:28:09,929 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5964e7cc817c54e7 with lease ID 0xba9d9ca3b0b8becd: from storage DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd node DatanodeRegistration(127.0.0.1:41475, datanodeUuid=6e5f6fb1-d15a-4191-9fe6-f53a63d1617f, infoPort=35855, infoSecurePort=0, ipcPort=46841, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:28:09,929 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5964e7cc817c54e7 with lease ID 0xba9d9ca3b0b8becd: Processing first storage report for DS-fcb94ffa-f3e9-4390-899a-2310911f1511 from datanode DatanodeRegistration(127.0.0.1:41475, datanodeUuid=6e5f6fb1-d15a-4191-9fe6-f53a63d1617f, infoPort=35855, infoSecurePort=0, ipcPort=46841, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764)
2024-12-06T05:28:09,929 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5964e7cc817c54e7 with lease ID 0xba9d9ca3b0b8becd: from storage DS-fcb94ffa-f3e9-4390-899a-2310911f1511 node DatanodeRegistration(127.0.0.1:41475, datanodeUuid=6e5f6fb1-d15a-4191-9fe6-f53a63d1617f, infoPort=35855, infoSecurePort=0, ipcPort=46841, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:28:10,041 WARN  [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data3/current/BP-2129024424-172.17.0.3-1733462887764/current, will proceed with Du for space computation calculation, 
2024-12-06T05:28:10,041 WARN  [Thread-685 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data4/current/BP-2129024424-172.17.0.3-1733462887764/current, will proceed with Du for space computation calculation, 
2024-12-06T05:28:10,056 WARN  [Thread-660 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:28:10,059 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2f3f44c0fc3aaaac with lease ID 0xba9d9ca3b0b8bece: Processing first storage report for DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96 from datanode DatanodeRegistration(127.0.0.1:44389, datanodeUuid=e97cdf8a-79a9-4085-ba36-a94e2f6ff519, infoPort=46643, infoSecurePort=0, ipcPort=37917, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764)
2024-12-06T05:28:10,059 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f3f44c0fc3aaaac with lease ID 0xba9d9ca3b0b8bece: from storage DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96 node DatanodeRegistration(127.0.0.1:44389, datanodeUuid=e97cdf8a-79a9-4085-ba36-a94e2f6ff519, infoPort=46643, infoSecurePort=0, ipcPort=37917, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:28:10,059 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2f3f44c0fc3aaaac with lease ID 0xba9d9ca3b0b8bece: Processing first storage report for DS-86c9fffd-f083-42a7-b3d5-009b11562dea from datanode DatanodeRegistration(127.0.0.1:44389, datanodeUuid=e97cdf8a-79a9-4085-ba36-a94e2f6ff519, infoPort=46643, infoSecurePort=0, ipcPort=37917, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764)
2024-12-06T05:28:10,060 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f3f44c0fc3aaaac with lease ID 0xba9d9ca3b0b8bece: from storage DS-86c9fffd-f083-42a7-b3d5-009b11562dea node DatanodeRegistration(127.0.0.1:44389, datanodeUuid=e97cdf8a-79a9-4085-ba36-a94e2f6ff519, infoPort=46643, infoSecurePort=0, ipcPort=37917, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:28:10,086 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642
2024-12-06T05:28:10,090 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/zookeeper_0, clientPort=55669, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-06T05:28:10,091 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55669
2024-12-06T05:28:10,091 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:10,093 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:10,105 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41475 is added to blk_1073741825_1001 (size=7)
2024-12-06T05:28:10,105 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741825_1001 (size=7)
2024-12-06T05:28:10,107 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086 with version=8
2024-12-06T05:28:10,107 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/hbase-staging
2024-12-06T05:28:10,110 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/ec46afd49254:0 server-side Connection retries=45
2024-12-06T05:28:10,110 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:28:10,111 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T05:28:10,111 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T05:28:10,111 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:28:10,111 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T05:28:10,111 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-06T05:28:10,111 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T05:28:10,112 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34167
2024-12-06T05:28:10,114 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34167 connecting to ZooKeeper ensemble=127.0.0.1:55669
2024-12-06T05:28:10,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:341670x0, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T05:28:10,211 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34167-0x101a91c92140000 connected
2024-12-06T05:28:10,446 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:10,449 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:10,453 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:28:10,453 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086, hbase.cluster.distributed=false
2024-12-06T05:28:10,455 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T05:28:10,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34167
2024-12-06T05:28:10,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34167
2024-12-06T05:28:10,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34167
2024-12-06T05:28:10,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34167
2024-12-06T05:28:10,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34167
2024-12-06T05:28:10,475 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/ec46afd49254:0 server-side Connection retries=45
2024-12-06T05:28:10,475 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:28:10,475 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T05:28:10,475 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T05:28:10,476 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:28:10,476 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T05:28:10,476 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-06T05:28:10,476 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T05:28:10,476 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39957
2024-12-06T05:28:10,478 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39957 connecting to ZooKeeper ensemble=127.0.0.1:55669
2024-12-06T05:28:10,478 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:10,480 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:10,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:399570x0, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T05:28:10,488 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:28:10,488 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39957-0x101a91c92140001 connected
2024-12-06T05:28:10,488 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-06T05:28:10,489 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-06T05:28:10,489 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-06T05:28:10,490 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T05:28:10,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39957
2024-12-06T05:28:10,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39957
2024-12-06T05:28:10,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39957
2024-12-06T05:28:10,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39957
2024-12-06T05:28:10,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39957
2024-12-06T05:28:10,503 DEBUG [M:0;ec46afd49254:34167 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ec46afd49254:34167
2024-12-06T05:28:10,504 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ec46afd49254,34167,1733462890110
2024-12-06T05:28:10,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:28:10,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:28:10,517 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ec46afd49254,34167,1733462890110
2024-12-06T05:28:10,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-06T05:28:10,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:10,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:10,530 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-06T05:28:10,530 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ec46afd49254,34167,1733462890110 from backup master directory
2024-12-06T05:28:10,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ec46afd49254,34167,1733462890110
2024-12-06T05:28:10,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:28:10,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:28:10,540 WARN  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T05:28:10,540 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ec46afd49254,34167,1733462890110
2024-12-06T05:28:10,546 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/hbase.id] with ID: 7258981e-378a-4920-8a80-a05fb2d41581
2024-12-06T05:28:10,546 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/.tmp/hbase.id
2024-12-06T05:28:10,553 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41475 is added to blk_1073741826_1002 (size=42)
2024-12-06T05:28:10,553 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741826_1002 (size=42)
2024-12-06T05:28:10,554 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/.tmp/hbase.id]:[hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/hbase.id]
2024-12-06T05:28:10,569 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:10,569 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-06T05:28:10,571 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-12-06T05:28:10,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:10,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:10,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41475 is added to blk_1073741827_1003 (size=196)
2024-12-06T05:28:10,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741827_1003 (size=196)
2024-12-06T05:28:10,650 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T05:28:10,651 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-06T05:28:10,651 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:28:10,659 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41475 is added to blk_1073741828_1004 (size=1189)
2024-12-06T05:28:10,659 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741828_1004 (size=1189)
2024-12-06T05:28:10,660 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store
2024-12-06T05:28:10,668 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741829_1005 (size=34)
2024-12-06T05:28:10,668 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41475 is added to blk_1073741829_1005 (size=34)
2024-12-06T05:28:10,669 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:28:10,669 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T05:28:10,669 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:10,669 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:10,669 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T05:28:10,669 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:10,669 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:10,669 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733462890669Disabling compacts and flushes for region at 1733462890669Disabling writes for close at 1733462890669Writing region close event to WAL at 1733462890669Closed at 1733462890669
2024-12-06T05:28:10,670 WARN  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/.initializing
2024-12-06T05:28:10,670 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/WALs/ec46afd49254,34167,1733462890110
2024-12-06T05:28:10,673 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C34167%2C1733462890110, suffix=, logDir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/WALs/ec46afd49254,34167,1733462890110, archiveDir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/oldWALs, maxLogs=10
2024-12-06T05:28:10,674 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C34167%2C1733462890110.1733462890673
2024-12-06T05:28:10,680 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/WALs/ec46afd49254,34167,1733462890110/ec46afd49254%2C34167%2C1733462890110.1733462890673
2024-12-06T05:28:10,688 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35855:35855),(127.0.0.1/127.0.0.1:46643:46643)]
2024-12-06T05:28:10,688 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:28:10,689 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:28:10,689 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:28:10,689 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:28:10,691 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:28:10,693 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-06T05:28:10,693 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:10,693 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:10,693 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:28:10,695 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-06T05:28:10,695 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:10,695 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:28:10,696 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:28:10,697 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-06T05:28:10,697 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:10,698 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:28:10,698 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:28:10,699 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-06T05:28:10,699 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:10,700 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:28:10,700 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:28:10,701 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:28:10,701 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:28:10,702 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:28:10,702 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:28:10,703 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-06T05:28:10,704 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:28:10,707 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:28:10,707 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=796631, jitterRate=0.012969374656677246}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-06T05:28:10,708 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733462890689Initializing all the Stores at 1733462890690 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462890690Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462890691 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462890691Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462890691Cleaning up temporary data from old regions at 1733462890703 (+12 ms)Region opened successfully at 1733462890708 (+5 ms)
2024-12-06T05:28:10,712 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-06T05:28:10,716 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36744030, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0
2024-12-06T05:28:10,718 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-06T05:28:10,718 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-06T05:28:10,718 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-06T05:28:10,718 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-06T05:28:10,719 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-06T05:28:10,719 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-06T05:28:10,719 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-06T05:28:10,722 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-06T05:28:10,723 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-06T05:28:10,762 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-06T05:28:10,762 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-06T05:28:10,763 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-06T05:28:10,836 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-06T05:28:10,837 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-06T05:28:10,838 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-06T05:28:10,903 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-06T05:28:10,905 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-06T05:28:10,974 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-06T05:28:10,982 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-06T05:28:11,003 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-06T05:28:11,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T05:28:11,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T05:28:11,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:11,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:11,074 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ec46afd49254,34167,1733462890110, sessionid=0x101a91c92140000, setting cluster-up flag (Was=false)
2024-12-06T05:28:11,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:11,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:11,291 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-06T05:28:11,295 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec46afd49254,34167,1733462890110
2024-12-06T05:28:11,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:11,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:11,487 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-06T05:28:11,489 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec46afd49254,34167,1733462890110
2024-12-06T05:28:11,491 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-06T05:28:11,493 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-06T05:28:11,494 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-06T05:28:11,494 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-06T05:28:11,494 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ec46afd49254,34167,1733462890110
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-06T05:28:11,496 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(746): ClusterId : 7258981e-378a-4920-8a80-a05fb2d41581
2024-12-06T05:28:11,496 DEBUG [RS:0;ec46afd49254:39957 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-06T05:28:11,496 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:28:11,496 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:28:11,497 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:28:11,497 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:28:11,497 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ec46afd49254:0, corePoolSize=10, maxPoolSize=10
2024-12-06T05:28:11,497 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:11,497 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ec46afd49254:0, corePoolSize=2, maxPoolSize=2
2024-12-06T05:28:11,497 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:11,500 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733462921500
2024-12-06T05:28:11,500 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-06T05:28:11,500 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-06T05:28:11,500 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-06T05:28:11,500 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-06T05:28:11,500 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-06T05:28:11,500 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-06T05:28:11,500 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:11,501 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-06T05:28:11,501 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:28:11,501 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-06T05:28:11,501 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-06T05:28:11,501 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-06T05:28:11,502 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-06T05:28:11,502 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-06T05:28:11,503 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462891502,5,FailOnTimeoutGroup]
2024-12-06T05:28:11,503 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:11,503 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-06T05:28:11,508 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462891503,5,FailOnTimeoutGroup]
2024-12-06T05:28:11,508 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:11,508 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-06T05:28:11,508 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:11,509 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:11,517 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741831_1007 (size=1321)
2024-12-06T05:28:11,517 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41475 is added to blk_1073741831_1007 (size=1321)
2024-12-06T05:28:11,518 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-06T05:28:11,518 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086
2024-12-06T05:28:11,520 DEBUG [RS:0;ec46afd49254:39957 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-06T05:28:11,520 DEBUG [RS:0;ec46afd49254:39957 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-06T05:28:11,526 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41475 is added to blk_1073741832_1008 (size=32)
2024-12-06T05:28:11,526 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741832_1008 (size=32)
2024-12-06T05:28:11,527 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:28:11,528 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T05:28:11,530 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T05:28:11,530 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:11,530 DEBUG [RS:0;ec46afd49254:39957 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-06T05:28:11,531 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:11,531 DEBUG [RS:0;ec46afd49254:39957 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae12d46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0
2024-12-06T05:28:11,531 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T05:28:11,532 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T05:28:11,532 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:11,532 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:11,533 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T05:28:11,534 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T05:28:11,534 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:11,535 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:11,535 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T05:28:11,536 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T05:28:11,536 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:11,537 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:11,537 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T05:28:11,538 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740
2024-12-06T05:28:11,538 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740
2024-12-06T05:28:11,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T05:28:11,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T05:28:11,540 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T05:28:11,541 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T05:28:11,543 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:28:11,544 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=794612, jitterRate=0.010401546955108643}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T05:28:11,544 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733462891527Initializing all the Stores at 1733462891528 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462891528Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462891528Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462891528Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462891528Cleaning up temporary data from old regions at 1733462891540 (+12 ms)Region opened successfully at 1733462891544 (+4 ms)
2024-12-06T05:28:11,544 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T05:28:11,545 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T05:28:11,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T05:28:11,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T05:28:11,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T05:28:11,545 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T05:28:11,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733462891544Disabling compacts and flushes for region at 1733462891544Disabling writes for close at 1733462891545 (+1 ms)Writing region close event to WAL at 1733462891545Closed at 1733462891545
2024-12-06T05:28:11,546 DEBUG [RS:0;ec46afd49254:39957 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ec46afd49254:39957
2024-12-06T05:28:11,546 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-06T05:28:11,546 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-06T05:28:11,546 DEBUG [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-06T05:28:11,547 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec46afd49254,34167,1733462890110 with port=39957, startcode=1733462890475
2024-12-06T05:28:11,547 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:28:11,547 DEBUG [RS:0;ec46afd49254:39957 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-06T05:28:11,547 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-06T05:28:11,547 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-06T05:28:11,549 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T05:28:11,549 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42977, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-12-06T05:28:11,549 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34167 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ec46afd49254,39957,1733462890475
2024-12-06T05:28:11,549 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34167 {}] master.ServerManager(517): Registering regionserver=ec46afd49254,39957,1733462890475
2024-12-06T05:28:11,550 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-06T05:28:11,551 DEBUG [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086
2024-12-06T05:28:11,551 DEBUG [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41215
2024-12-06T05:28:11,551 DEBUG [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-06T05:28:11,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T05:28:11,611 DEBUG [RS:0;ec46afd49254:39957 {}] zookeeper.ZKUtil(111): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec46afd49254,39957,1733462890475
2024-12-06T05:28:11,611 WARN  [RS:0;ec46afd49254:39957 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T05:28:11,611 INFO  [RS:0;ec46afd49254:39957 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:28:11,611 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec46afd49254,39957,1733462890475]
2024-12-06T05:28:11,612 DEBUG [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475
2024-12-06T05:28:11,620 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-06T05:28:11,642 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-06T05:28:11,644 INFO  [RS:0;ec46afd49254:39957 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T05:28:11,644 INFO  [RS:0;ec46afd49254:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:11,648 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-06T05:28:11,649 INFO  [RS:0;ec46afd49254:39957 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-06T05:28:11,649 INFO  [RS:0;ec46afd49254:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:11,649 DEBUG [RS:0;ec46afd49254:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:11,649 DEBUG [RS:0;ec46afd49254:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:11,649 DEBUG [RS:0;ec46afd49254:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:11,649 DEBUG [RS:0;ec46afd49254:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:11,649 DEBUG [RS:0;ec46afd49254:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:11,649 DEBUG [RS:0;ec46afd49254:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0, corePoolSize=2, maxPoolSize=2
2024-12-06T05:28:11,649 DEBUG [RS:0;ec46afd49254:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:11,649 DEBUG [RS:0;ec46afd49254:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:11,649 DEBUG [RS:0;ec46afd49254:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:11,650 DEBUG [RS:0;ec46afd49254:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:11,650 DEBUG [RS:0;ec46afd49254:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:11,650 DEBUG [RS:0;ec46afd49254:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:11,650 DEBUG [RS:0;ec46afd49254:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3
2024-12-06T05:28:11,650 DEBUG [RS:0;ec46afd49254:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3
2024-12-06T05:28:11,650 INFO  [RS:0;ec46afd49254:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:11,650 INFO  [RS:0;ec46afd49254:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:11,650 INFO  [RS:0;ec46afd49254:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:11,650 INFO  [RS:0;ec46afd49254:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:11,650 INFO  [RS:0;ec46afd49254:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:11,650 INFO  [RS:0;ec46afd49254:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,39957,1733462890475-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T05:28:11,663 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-06T05:28:11,664 INFO  [RS:0;ec46afd49254:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,39957,1733462890475-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:11,664 INFO  [RS:0;ec46afd49254:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:11,664 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.Replication(171): ec46afd49254,39957,1733462890475 started
2024-12-06T05:28:11,676 INFO  [RS:0;ec46afd49254:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:11,676 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(1482): Serving as ec46afd49254,39957,1733462890475, RpcServer on ec46afd49254/172.17.0.3:39957, sessionid=0x101a91c92140001
2024-12-06T05:28:11,676 DEBUG [RS:0;ec46afd49254:39957 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-06T05:28:11,676 DEBUG [RS:0;ec46afd49254:39957 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec46afd49254,39957,1733462890475
2024-12-06T05:28:11,676 DEBUG [RS:0;ec46afd49254:39957 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,39957,1733462890475'
2024-12-06T05:28:11,676 DEBUG [RS:0;ec46afd49254:39957 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-06T05:28:11,677 DEBUG [RS:0;ec46afd49254:39957 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-06T05:28:11,678 DEBUG [RS:0;ec46afd49254:39957 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-06T05:28:11,678 DEBUG [RS:0;ec46afd49254:39957 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-06T05:28:11,678 DEBUG [RS:0;ec46afd49254:39957 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec46afd49254,39957,1733462890475
2024-12-06T05:28:11,678 DEBUG [RS:0;ec46afd49254:39957 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,39957,1733462890475'
2024-12-06T05:28:11,678 DEBUG [RS:0;ec46afd49254:39957 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-06T05:28:11,678 DEBUG [RS:0;ec46afd49254:39957 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-06T05:28:11,679 DEBUG [RS:0;ec46afd49254:39957 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-06T05:28:11,679 INFO  [RS:0;ec46afd49254:39957 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-06T05:28:11,679 INFO  [RS:0;ec46afd49254:39957 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-06T05:28:11,700 WARN  [ec46afd49254:34167 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-06T05:28:11,782 INFO  [RS:0;ec46afd49254:39957 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C39957%2C1733462890475, suffix=, logDir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475, archiveDir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/oldWALs, maxLogs=32
2024-12-06T05:28:11,782 INFO  [RS:0;ec46afd49254:39957 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C39957%2C1733462890475.1733462891782
2024-12-06T05:28:11,795 INFO  [RS:0;ec46afd49254:39957 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462891782
2024-12-06T05:28:11,796 DEBUG [RS:0;ec46afd49254:39957 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35855:35855),(127.0.0.1/127.0.0.1:46643:46643)]
2024-12-06T05:28:11,950 DEBUG [ec46afd49254:34167 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-06T05:28:11,951 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ec46afd49254,39957,1733462890475
2024-12-06T05:28:11,953 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec46afd49254,39957,1733462890475, state=OPENING
2024-12-06T05:28:11,961 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-06T05:28:11,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:11,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:11,973 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T05:28:11,973 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:28:11,974 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:28:11,974 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ec46afd49254,39957,1733462890475}]
2024-12-06T05:28:12,130 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-06T05:28:12,135 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46623, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-06T05:28:12,142 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-06T05:28:12,142 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:28:12,144 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C39957%2C1733462890475.meta, suffix=.meta, logDir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475, archiveDir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/oldWALs, maxLogs=32
2024-12-06T05:28:12,145 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
2024-12-06T05:28:12,151 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
2024-12-06T05:28:12,152 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35855:35855),(127.0.0.1/127.0.0.1:46643:46643)]
2024-12-06T05:28:12,153 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:28:12,153 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-06T05:28:12,153 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-06T05:28:12,153 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-06T05:28:12,153 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-06T05:28:12,153 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:28:12,153 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-06T05:28:12,154 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-06T05:28:12,158 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T05:28:12,159 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T05:28:12,159 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:12,159 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:12,160 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T05:28:12,160 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T05:28:12,160 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:12,161 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:12,161 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T05:28:12,162 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T05:28:12,162 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:12,162 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:12,163 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T05:28:12,163 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T05:28:12,163 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:12,164 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:28:12,164 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T05:28:12,165 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740
2024-12-06T05:28:12,166 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740
2024-12-06T05:28:12,167 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T05:28:12,167 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T05:28:12,168 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T05:28:12,169 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T05:28:12,170 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=709630, jitterRate=-0.09765921533107758}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T05:28:12,170 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-06T05:28:12,171 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733462892154Writing region info on filesystem at 1733462892154Initializing all the Stores at 1733462892155 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462892155Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462892157 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462892157Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462892157Cleaning up temporary data from old regions at 1733462892167 (+10 ms)Running coprocessor post-open hooks at 1733462892170 (+3 ms)Region opened successfully at 1733462892170
2024-12-06T05:28:12,172 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733462892129
2024-12-06T05:28:12,174 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-06T05:28:12,174 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-06T05:28:12,175 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ec46afd49254,39957,1733462890475
2024-12-06T05:28:12,176 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec46afd49254,39957,1733462890475, state=OPEN
2024-12-06T05:28:12,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T05:28:12,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T05:28:12,211 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ec46afd49254,39957,1733462890475
2024-12-06T05:28:12,212 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:28:12,212 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:28:12,217 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-06T05:28:12,217 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec46afd49254,39957,1733462890475 in 239 msec
2024-12-06T05:28:12,221 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-06T05:28:12,221 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 670 msec
2024-12-06T05:28:12,222 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:28:12,222 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-06T05:28:12,224 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T05:28:12,224 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec46afd49254,39957,1733462890475, seqNum=-1]
2024-12-06T05:28:12,224 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T05:28:12,225 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59229, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T05:28:12,232 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 738 msec
2024-12-06T05:28:12,232 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733462892232, completionTime=-1
2024-12-06T05:28:12,232 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-06T05:28:12,232 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-06T05:28:12,234 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-06T05:28:12,234 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733462952234
2024-12-06T05:28:12,234 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733463012234
2024-12-06T05:28:12,234 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-06T05:28:12,235 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,34167,1733462890110-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:12,235 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,34167,1733462890110-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:12,235 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,34167,1733462890110-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:12,235 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ec46afd49254:34167, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:12,235 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:12,235 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:12,237 DEBUG [master/ec46afd49254:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T05:28:12,238 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.698sec
2024-12-06T05:28:12,239 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-06T05:28:12,239 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-06T05:28:12,239 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-06T05:28:12,239 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-06T05:28:12,239 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-06T05:28:12,239 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,34167,1733462890110-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T05:28:12,239 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,34167,1733462890110-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-06T05:28:12,242 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-06T05:28:12,242 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-06T05:28:12,242 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,34167,1733462890110-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:12,297 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794a0c48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:28:12,297 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ec46afd49254,34167,-1 for getting cluster id
2024-12-06T05:28:12,297 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-06T05:28:12,300 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7258981e-378a-4920-8a80-a05fb2d41581'
2024-12-06T05:28:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-06T05:28:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7258981e-378a-4920-8a80-a05fb2d41581"

2024-12-06T05:28:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4429846e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:28:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ec46afd49254,34167,-1]
2024-12-06T05:28:12,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-06T05:28:12,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:12,304 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-06T05:28:12,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6702499c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:28:12,306 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T05:28:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec46afd49254,39957,1733462890475, seqNum=-1]
2024-12-06T05:28:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T05:28:12,310 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T05:28:12,312 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ec46afd49254,34167,1733462890110
2024-12-06T05:28:12,312 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:12,315 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-06T05:28:12,329 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/ec46afd49254:0 server-side Connection retries=45
2024-12-06T05:28:12,329 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:28:12,329 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T05:28:12,329 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T05:28:12,329 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:28:12,329 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T05:28:12,329 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-06T05:28:12,329 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T05:28:12,330 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34771
2024-12-06T05:28:12,331 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34771 connecting to ZooKeeper ensemble=127.0.0.1:55669
2024-12-06T05:28:12,332 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:12,333 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:28:12,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:347710x0, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T05:28:12,349 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:34771-0x101a91c92140002, quorum=127.0.0.1:55669, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-12-06T05:28:12,349 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-12-06T05:28:12,349 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34771-0x101a91c92140002 connected
2024-12-06T05:28:12,350 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-06T05:28:12,351 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-06T05:28:12,351 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:34771-0x101a91c92140002, quorum=127.0.0.1:55669, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-06T05:28:12,353 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34771-0x101a91c92140002, quorum=127.0.0.1:55669, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T05:28:12,354 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34771
2024-12-06T05:28:12,354 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34771
2024-12-06T05:28:12,355 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34771
2024-12-06T05:28:12,355 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34771
2024-12-06T05:28:12,355 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34771
2024-12-06T05:28:12,357 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.HRegionServer(746): ClusterId : 7258981e-378a-4920-8a80-a05fb2d41581
2024-12-06T05:28:12,357 DEBUG [RS:1;ec46afd49254:34771 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-06T05:28:12,379 DEBUG [RS:1;ec46afd49254:34771 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-06T05:28:12,379 DEBUG [RS:1;ec46afd49254:34771 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-06T05:28:12,393 DEBUG [RS:1;ec46afd49254:34771 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-06T05:28:12,394 DEBUG [RS:1;ec46afd49254:34771 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@146d41d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0
2024-12-06T05:28:12,405 DEBUG [RS:1;ec46afd49254:34771 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ec46afd49254:34771
2024-12-06T05:28:12,405 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-06T05:28:12,406 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-06T05:28:12,406 DEBUG [RS:1;ec46afd49254:34771 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-06T05:28:12,406 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec46afd49254,34167,1733462890110 with port=34771, startcode=1733462892328
2024-12-06T05:28:12,406 DEBUG [RS:1;ec46afd49254:34771 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-06T05:28:12,408 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43385, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-12-06T05:28:12,409 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34167 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ec46afd49254,34771,1733462892328
2024-12-06T05:28:12,409 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34167 {}] master.ServerManager(517): Registering regionserver=ec46afd49254,34771,1733462892328
2024-12-06T05:28:12,411 DEBUG [RS:1;ec46afd49254:34771 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086
2024-12-06T05:28:12,411 DEBUG [RS:1;ec46afd49254:34771 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41215
2024-12-06T05:28:12,411 DEBUG [RS:1;ec46afd49254:34771 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-06T05:28:12,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T05:28:12,425 DEBUG [RS:1;ec46afd49254:34771 {}] zookeeper.ZKUtil(111): regionserver:34771-0x101a91c92140002, quorum=127.0.0.1:55669, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec46afd49254,34771,1733462892328
2024-12-06T05:28:12,425 WARN  [RS:1;ec46afd49254:34771 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T05:28:12,425 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec46afd49254,34771,1733462892328]
2024-12-06T05:28:12,425 INFO  [RS:1;ec46afd49254:34771 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:28:12,426 DEBUG [RS:1;ec46afd49254:34771 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328
2024-12-06T05:28:12,431 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-06T05:28:12,432 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-06T05:28:12,436 INFO  [RS:1;ec46afd49254:34771 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T05:28:12,437 INFO  [RS:1;ec46afd49254:34771 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:12,437 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-06T05:28:12,438 INFO  [RS:1;ec46afd49254:34771 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-06T05:28:12,438 INFO  [RS:1;ec46afd49254:34771 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:12,439 DEBUG [RS:1;ec46afd49254:34771 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:12,439 DEBUG [RS:1;ec46afd49254:34771 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:12,439 DEBUG [RS:1;ec46afd49254:34771 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:12,439 DEBUG [RS:1;ec46afd49254:34771 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:12,439 DEBUG [RS:1;ec46afd49254:34771 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:12,439 DEBUG [RS:1;ec46afd49254:34771 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0, corePoolSize=2, maxPoolSize=2
2024-12-06T05:28:12,439 DEBUG [RS:1;ec46afd49254:34771 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:12,439 DEBUG [RS:1;ec46afd49254:34771 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:12,439 DEBUG [RS:1;ec46afd49254:34771 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:12,439 DEBUG [RS:1;ec46afd49254:34771 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:12,439 DEBUG [RS:1;ec46afd49254:34771 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:12,439 DEBUG [RS:1;ec46afd49254:34771 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:28:12,439 DEBUG [RS:1;ec46afd49254:34771 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3
2024-12-06T05:28:12,439 DEBUG [RS:1;ec46afd49254:34771 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3
2024-12-06T05:28:12,440 INFO  [RS:1;ec46afd49254:34771 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:12,440 INFO  [RS:1;ec46afd49254:34771 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:12,440 INFO  [RS:1;ec46afd49254:34771 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:12,440 INFO  [RS:1;ec46afd49254:34771 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:12,440 INFO  [RS:1;ec46afd49254:34771 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:12,440 INFO  [RS:1;ec46afd49254:34771 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,34771,1733462892328-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T05:28:12,453 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-06T05:28:12,453 INFO  [RS:1;ec46afd49254:34771 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,34771,1733462892328-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:12,453 INFO  [RS:1;ec46afd49254:34771 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:12,453 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.Replication(171): ec46afd49254,34771,1733462892328 started
2024-12-06T05:28:12,465 INFO  [RS:1;ec46afd49254:34771 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:28:12,465 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.HRegionServer(1482): Serving as ec46afd49254,34771,1733462892328, RpcServer on ec46afd49254/172.17.0.3:34771, sessionid=0x101a91c92140002
2024-12-06T05:28:12,465 DEBUG [RS:1;ec46afd49254:34771 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-06T05:28:12,465 DEBUG [RS:1;ec46afd49254:34771 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec46afd49254,34771,1733462892328
2024-12-06T05:28:12,465 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;ec46afd49254:34771,5,FailOnTimeoutGroup]
2024-12-06T05:28:12,465 DEBUG [RS:1;ec46afd49254:34771 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,34771,1733462892328'
2024-12-06T05:28:12,465 DEBUG [RS:1;ec46afd49254:34771 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-06T05:28:12,466 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-12-06T05:28:12,466 DEBUG [RS:1;ec46afd49254:34771 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-06T05:28:12,466 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-06T05:28:12,466 DEBUG [RS:1;ec46afd49254:34771 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-06T05:28:12,466 DEBUG [RS:1;ec46afd49254:34771 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-06T05:28:12,467 DEBUG [RS:1;ec46afd49254:34771 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec46afd49254,34771,1733462892328
2024-12-06T05:28:12,467 DEBUG [RS:1;ec46afd49254:34771 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,34771,1733462892328'
2024-12-06T05:28:12,467 DEBUG [RS:1;ec46afd49254:34771 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-06T05:28:12,467 DEBUG [RS:1;ec46afd49254:34771 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-06T05:28:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is ec46afd49254,34167,1733462890110
2024-12-06T05:28:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2a7612cc
2024-12-06T05:28:12,467 DEBUG [RS:1;ec46afd49254:34771 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-06T05:28:12,467 INFO  [RS:1;ec46afd49254:34771 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-06T05:28:12,467 INFO  [RS:1;ec46afd49254:34771 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-06T05:28:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-06T05:28:12,469 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-06T05:28:12,470 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34167 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-06T05:28:12,470 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34167 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-06T05:28:12,470 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34167 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T05:28:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34167 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-12-06T05:28:12,472 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-12-06T05:28:12,473 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:12,473 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34167 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-12-06T05:28:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34167 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T05:28:12,474 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-06T05:28:12,481 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41475 is added to blk_1073741835_1011 (size=393)
2024-12-06T05:28:12,482 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741835_1011 (size=393)
2024-12-06T05:28:12,483 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a9e4791af9aa7cd7b9961aee9a47e3a5, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086
2024-12-06T05:28:12,491 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741836_1012 (size=76)
2024-12-06T05:28:12,491 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41475 is added to blk_1073741836_1012 (size=76)
2024-12-06T05:28:12,491 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:28:12,492 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing a9e4791af9aa7cd7b9961aee9a47e3a5, disabling compactions & flushes
2024-12-06T05:28:12,492 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.
2024-12-06T05:28:12,492 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.
2024-12-06T05:28:12,492 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5. after waiting 0 ms
2024-12-06T05:28:12,492 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.
2024-12-06T05:28:12,492 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.
2024-12-06T05:28:12,492 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for a9e4791af9aa7cd7b9961aee9a47e3a5:
Waiting for close lock at 1733462892492Disabling compacts and flushes for region at 1733462892492Disabling writes for close at 1733462892492Writing region close event to WAL at 1733462892492Closed at 1733462892492
2024-12-06T05:28:12,494 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-12-06T05:28:12,494 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1733462892494"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733462892494"}]},"ts":"1733462892494"}
2024-12-06T05:28:12,497 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-06T05:28:12,498 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-06T05:28:12,498 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733462892498"}]},"ts":"1733462892498"}
2024-12-06T05:28:12,500 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-12-06T05:28:12,501 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=a9e4791af9aa7cd7b9961aee9a47e3a5, ASSIGN}]
2024-12-06T05:28:12,502 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=a9e4791af9aa7cd7b9961aee9a47e3a5, ASSIGN
2024-12-06T05:28:12,503 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=a9e4791af9aa7cd7b9961aee9a47e3a5, ASSIGN; state=OFFLINE, location=ec46afd49254,39957,1733462890475; forceNewPlan=false, retain=false
2024-12-06T05:28:12,573 INFO  [RS:1;ec46afd49254:34771 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C34771%2C1733462892328, suffix=, logDir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328, archiveDir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/oldWALs, maxLogs=32
2024-12-06T05:28:12,576 INFO  [RS:1;ec46afd49254:34771 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C34771%2C1733462892328.1733462892575
2024-12-06T05:28:12,582 INFO  [RS:1;ec46afd49254:34771 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
2024-12-06T05:28:12,583 DEBUG [RS:1;ec46afd49254:34771 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35855:35855),(127.0.0.1/127.0.0.1:46643:46643)]
2024-12-06T05:28:12,654 INFO  [ec46afd49254:34167 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-12-06T05:28:12,655 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a9e4791af9aa7cd7b9961aee9a47e3a5, regionState=OPENING, regionLocation=ec46afd49254,39957,1733462890475
2024-12-06T05:28:12,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=a9e4791af9aa7cd7b9961aee9a47e3a5, ASSIGN because future has completed
2024-12-06T05:28:12,659 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9e4791af9aa7cd7b9961aee9a47e3a5, server=ec46afd49254,39957,1733462890475}]
2024-12-06T05:28:12,825 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.
2024-12-06T05:28:12,826 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a9e4791af9aa7cd7b9961aee9a47e3a5, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:28:12,826 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:12,826 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:28:12,826 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:12,827 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:12,828 INFO  [StoreOpener-a9e4791af9aa7cd7b9961aee9a47e3a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region a9e4791af9aa7cd7b9961aee9a47e3a5 
2024-12-06T05:28:12,830 INFO  [StoreOpener-a9e4791af9aa7cd7b9961aee9a47e3a5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9e4791af9aa7cd7b9961aee9a47e3a5 columnFamilyName info
2024-12-06T05:28:12,830 DEBUG [StoreOpener-a9e4791af9aa7cd7b9961aee9a47e3a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:28:12,831 INFO  [StoreOpener-a9e4791af9aa7cd7b9961aee9a47e3a5-1 {}] regionserver.HStore(327): Store=a9e4791af9aa7cd7b9961aee9a47e3a5/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:28:12,831 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:12,832 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:12,832 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:12,833 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:12,833 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:12,834 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:12,836 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:28:12,837 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a9e4791af9aa7cd7b9961aee9a47e3a5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=703739, jitterRate=-0.10515052080154419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-06T05:28:12,837 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:12,837 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a9e4791af9aa7cd7b9961aee9a47e3a5:
Running coprocessor pre-open hook at 1733462892827Writing region info on filesystem at 1733462892827Initializing all the Stores at 1733462892828 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462892828Cleaning up temporary data from old regions at 1733462892833 (+5 ms)Running coprocessor post-open hooks at 1733462892837 (+4 ms)Region opened successfully at 1733462892837
2024-12-06T05:28:12,839 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5., pid=6, masterSystemTime=1733462892813
2024-12-06T05:28:12,841 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.
2024-12-06T05:28:12,841 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.
2024-12-06T05:28:12,842 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a9e4791af9aa7cd7b9961aee9a47e3a5, regionState=OPEN, openSeqNum=2, regionLocation=ec46afd49254,39957,1733462890475
2024-12-06T05:28:12,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9e4791af9aa7cd7b9961aee9a47e3a5, server=ec46afd49254,39957,1733462890475 because future has completed
2024-12-06T05:28:12,848 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-06T05:28:12,848 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a9e4791af9aa7cd7b9961aee9a47e3a5, server=ec46afd49254,39957,1733462890475 in 187 msec
2024-12-06T05:28:12,851 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-06T05:28:12,851 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=a9e4791af9aa7cd7b9961aee9a47e3a5, ASSIGN in 347 msec
2024-12-06T05:28:12,852 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-06T05:28:12,852 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733462892852"}]},"ts":"1733462892852"}
2024-12-06T05:28:12,855 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-12-06T05:28:12,856 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-12-06T05:28:12,859 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 386 msec
2024-12-06T05:28:16,505 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-06T05:28:16,506 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-06T05:28:16,509 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-06T05:28:16,509 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-12-06T05:28:16,510 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T05:28:16,510 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-06T05:28:17,656 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T05:28:17,659 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:17,680 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:17,682 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:17,683 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:17,693 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-12-06T05:28:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34167 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T05:28:22,498 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-12-06T05:28:22,498 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-12-06T05:28:22,506 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-12-06T05:28:22,506 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.
2024-12-06T05:28:22,518 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:28:22,521 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:28:22,521 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:28:22,521 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:28:22,521 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T05:28:22,522 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@65349436{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:28:22,522 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ab1ed71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:28:22,613 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6c5f7ee0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/java.io.tmpdir/jetty-localhost-39593-hadoop-hdfs-3_4_1-tests_jar-_-any-8522485195078177652/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:22,614 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@495a6aea{HTTP/1.1, (http/1.1)}{localhost:39593}
2024-12-06T05:28:22,614 INFO  [Time-limited test {}] server.Server(415): Started @122225ms
2024-12-06T05:28:22,615 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:28:22,641 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:28:22,644 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:28:22,644 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:28:22,644 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:28:22,644 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T05:28:22,645 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@514acf4b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:28:22,645 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7cde9b58{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:28:22,740 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6079f97a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/java.io.tmpdir/jetty-localhost-38015-hadoop-hdfs-3_4_1-tests_jar-_-any-18355501143728382870/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:22,740 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1196c8fc{HTTP/1.1, (http/1.1)}{localhost:38015}
2024-12-06T05:28:22,740 INFO  [Time-limited test {}] server.Server(415): Started @122351ms
2024-12-06T05:28:22,741 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:28:22,770 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:28:22,772 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:28:22,773 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:28:22,773 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:28:22,773 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T05:28:22,774 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@136e75a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:28:22,774 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@37673872{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:28:22,865 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1b65aae1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/java.io.tmpdir/jetty-localhost-46881-hadoop-hdfs-3_4_1-tests_jar-_-any-5280842442313109081/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:22,866 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e08dd81{HTTP/1.1, (http/1.1)}{localhost:46881}
2024-12-06T05:28:22,866 INFO  [Time-limited test {}] server.Server(415): Started @122477ms
2024-12-06T05:28:22,867 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:28:24,086 WARN  [Thread-868 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data5/current/BP-2129024424-172.17.0.3-1733462887764/current, will proceed with Du for space computation calculation, 
2024-12-06T05:28:24,086 WARN  [Thread-869 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data6/current/BP-2129024424-172.17.0.3-1733462887764/current, will proceed with Du for space computation calculation, 
2024-12-06T05:28:24,109 WARN  [Thread-809 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:28:24,111 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x36daf79d0ed53b3 with lease ID 0xba9d9ca3b0b8becf: Processing first storage report for DS-6c207a91-b278-477d-bd0b-5fb128ae6b31 from datanode DatanodeRegistration(127.0.0.1:42285, datanodeUuid=acf5f832-3abe-43fb-941b-af6d4505c21c, infoPort=40021, infoSecurePort=0, ipcPort=36209, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764)
2024-12-06T05:28:24,112 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x36daf79d0ed53b3 with lease ID 0xba9d9ca3b0b8becf: from storage DS-6c207a91-b278-477d-bd0b-5fb128ae6b31 node DatanodeRegistration(127.0.0.1:42285, datanodeUuid=acf5f832-3abe-43fb-941b-af6d4505c21c, infoPort=40021, infoSecurePort=0, ipcPort=36209, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:28:24,112 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x36daf79d0ed53b3 with lease ID 0xba9d9ca3b0b8becf: Processing first storage report for DS-9a335095-fabb-43cf-ae47-c330f470c4c2 from datanode DatanodeRegistration(127.0.0.1:42285, datanodeUuid=acf5f832-3abe-43fb-941b-af6d4505c21c, infoPort=40021, infoSecurePort=0, ipcPort=36209, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764)
2024-12-06T05:28:24,112 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x36daf79d0ed53b3 with lease ID 0xba9d9ca3b0b8becf: from storage DS-9a335095-fabb-43cf-ae47-c330f470c4c2 node DatanodeRegistration(127.0.0.1:42285, datanodeUuid=acf5f832-3abe-43fb-941b-af6d4505c21c, infoPort=40021, infoSecurePort=0, ipcPort=36209, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:28:24,199 WARN  [Thread-879 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data7/current/BP-2129024424-172.17.0.3-1733462887764/current, will proceed with Du for space computation calculation, 
2024-12-06T05:28:24,199 WARN  [Thread-880 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data8/current/BP-2129024424-172.17.0.3-1733462887764/current, will proceed with Du for space computation calculation, 
2024-12-06T05:28:24,215 WARN  [Thread-831 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:28:24,218 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc9ed2e3dcf294808 with lease ID 0xba9d9ca3b0b8bed0: Processing first storage report for DS-04112c23-8d9c-40a2-af3f-0f38711c8d43 from datanode DatanodeRegistration(127.0.0.1:41703, datanodeUuid=e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8, infoPort=39793, infoSecurePort=0, ipcPort=44537, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764)
2024-12-06T05:28:24,218 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc9ed2e3dcf294808 with lease ID 0xba9d9ca3b0b8bed0: from storage DS-04112c23-8d9c-40a2-af3f-0f38711c8d43 node DatanodeRegistration(127.0.0.1:41703, datanodeUuid=e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8, infoPort=39793, infoSecurePort=0, ipcPort=44537, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-06T05:28:24,218 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc9ed2e3dcf294808 with lease ID 0xba9d9ca3b0b8bed0: Processing first storage report for DS-75559295-d335-4a40-9aef-84aa46a9d186 from datanode DatanodeRegistration(127.0.0.1:41703, datanodeUuid=e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8, infoPort=39793, infoSecurePort=0, ipcPort=44537, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764)
2024-12-06T05:28:24,218 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc9ed2e3dcf294808 with lease ID 0xba9d9ca3b0b8bed0: from storage DS-75559295-d335-4a40-9aef-84aa46a9d186 node DatanodeRegistration(127.0.0.1:41703, datanodeUuid=e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8, infoPort=39793, infoSecurePort=0, ipcPort=44537, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:28:24,263 WARN  [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data9/current/BP-2129024424-172.17.0.3-1733462887764/current, will proceed with Du for space computation calculation, 
2024-12-06T05:28:24,263 WARN  [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data10/current/BP-2129024424-172.17.0.3-1733462887764/current, will proceed with Du for space computation calculation, 
2024-12-06T05:28:24,284 WARN  [Thread-853 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:28:24,286 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbbd8b245906e584e with lease ID 0xba9d9ca3b0b8bed1: Processing first storage report for DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa from datanode DatanodeRegistration(127.0.0.1:40573, datanodeUuid=bf1bb65e-9ed4-44da-b9e7-8b68590125bf, infoPort=38857, infoSecurePort=0, ipcPort=35499, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764)
2024-12-06T05:28:24,286 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbbd8b245906e584e with lease ID 0xba9d9ca3b0b8bed1: from storage DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa node DatanodeRegistration(127.0.0.1:40573, datanodeUuid=bf1bb65e-9ed4-44da-b9e7-8b68590125bf, infoPort=38857, infoSecurePort=0, ipcPort=35499, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:28:24,286 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbbd8b245906e584e with lease ID 0xba9d9ca3b0b8bed1: Processing first storage report for DS-b75bd13c-e93b-449b-999a-7c72e51716fc from datanode DatanodeRegistration(127.0.0.1:40573, datanodeUuid=bf1bb65e-9ed4-44da-b9e7-8b68590125bf, infoPort=38857, infoSecurePort=0, ipcPort=35499, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764)
2024-12-06T05:28:24,287 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbbd8b245906e584e with lease ID 0xba9d9ca3b0b8bed1: from storage DS-b75bd13c-e93b-449b-999a-7c72e51716fc node DatanodeRegistration(127.0.0.1:40573, datanodeUuid=bf1bb65e-9ed4-44da-b9e7-8b68590125bf, infoPort=38857, infoSecurePort=0, ipcPort=35499, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:28:24,306 WARN  [ResponseProcessor for block BP-2129024424-172.17.0.3-1733462887764:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2129024424-172.17.0.3-1733462887764:blk_1073741837_1013
java.io.IOException: Bad response ERROR for BP-2129024424-172.17.0.3-1733462887764:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:24,306 WARN  [ResponseProcessor for block BP-2129024424-172.17.0.3-1733462887764:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2129024424-172.17.0.3-1733462887764:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-2129024424-172.17.0.3-1733462887764:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:24,307 WARN  [ResponseProcessor for block BP-2129024424-172.17.0.3-1733462887764:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2129024424-172.17.0.3-1733462887764:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-2129024424-172.17.0.3-1733462887764:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:24,307 WARN  [DataStreamer for file /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575 block BP-2129024424-172.17.0.3-1733462887764:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK], DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]) is bad.
2024-12-06T05:28:24,307 WARN  [DataStreamer for file /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta block BP-2129024424-172.17.0.3-1733462887764:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK], DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]) is bad.
2024-12-06T05:28:24,307 WARN  [ResponseProcessor for block BP-2129024424-172.17.0.3-1733462887764:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2129024424-172.17.0.3-1733462887764:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-2129024424-172.17.0.3-1733462887764:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:24,306 WARN  [PacketResponder: BP-2129024424-172.17.0.3-1733462887764:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:44389] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:24,306 WARN  [PacketResponder: BP-2129024424-172.17.0.3-1733462887764:blk_1073741837_1013, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:44389] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:24,307 WARN  [PacketResponder: BP-2129024424-172.17.0.3-1733462887764:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:44389] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:24,307 WARN  [PacketResponder: BP-2129024424-172.17.0.3-1733462887764:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:44389] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:24,307 WARN  [DataStreamer for file /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/WALs/ec46afd49254,34167,1733462890110/ec46afd49254%2C34167%2C1733462890110.1733462890673 block BP-2129024424-172.17.0.3-1733462887764:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK], DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]) is bad.
2024-12-06T05:28:24,308 WARN  [DataStreamer for file /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462891782 block BP-2129024424-172.17.0.3-1733462887764:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK], DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]) is bad.
2024-12-06T05:28:24,308 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:55876 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41475:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55876 dst: /127.0.0.1:41475
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:24,308 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:41302 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41475:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41302 dst: /127.0.0.1:41475
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:24,308 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1189303528_22 at /127.0.0.1:45848 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:44389:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45848 dst: /127.0.0.1:44389
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:24,308 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:37674 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:44389:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37674 dst: /127.0.0.1:44389
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:24,308 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1391889316_22 at /127.0.0.1:55896 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:41475:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55896 dst: /127.0.0.1:41475
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:24,308 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:45882 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:44389:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45882 dst: /127.0.0.1:44389
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:24,308 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1189303528_22 at /127.0.0.1:41276 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41475:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41276 dst: /127.0.0.1:41475
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:24,308 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@48b0be64{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:24,308 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1391889316_22 at /127.0.0.1:37694 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:44389:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37694 dst: /127.0.0.1:44389
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:24,309 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4cae84f2{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:28:24,309 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:28:24,309 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2dc8ddff{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:28:24,309 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@57ebe64c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.log.dir/,STOPPED}
2024-12-06T05:28:24,310 WARN  [BP-2129024424-172.17.0.3-1733462887764 heartbeating to localhost/127.0.0.1:41215 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:28:24,310 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:28:24,310 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:28:24,310 WARN  [BP-2129024424-172.17.0.3-1733462887764 heartbeating to localhost/127.0.0.1:41215 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2129024424-172.17.0.3-1733462887764 (Datanode Uuid e97cdf8a-79a9-4085-ba36-a94e2f6ff519) service to localhost/127.0.0.1:41215
2024-12-06T05:28:24,311 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data3/current/BP-2129024424-172.17.0.3-1733462887764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:24,311 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data4/current/BP-2129024424-172.17.0.3-1733462887764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:24,311 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:28:24,312 WARN  [DataStreamer for file /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/WALs/ec46afd49254,34167,1733462890110/ec46afd49254%2C34167%2C1733462890110.1733462890673 block BP-2129024424-172.17.0.3-1733462887764:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:24,312 WARN  [DataStreamer for file /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575 block BP-2129024424-172.17.0.3-1733462887764:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:24,316 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5afc739a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:24,316 WARN  [DataStreamer for file /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462891782 block BP-2129024424-172.17.0.3-1733462887764:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:24,316 WARN  [DataStreamer for file /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta block BP-2129024424-172.17.0.3-1733462887764:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:24,317 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@200d0f88{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:28:24,317 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:28:24,317 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@21f2acf7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:28:24,317 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@180ba686{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.log.dir/,STOPPED}
2024-12-06T05:28:24,318 WARN  [BP-2129024424-172.17.0.3-1733462887764 heartbeating to localhost/127.0.0.1:41215 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:28:24,318 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:28:24,318 WARN  [BP-2129024424-172.17.0.3-1733462887764 heartbeating to localhost/127.0.0.1:41215 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2129024424-172.17.0.3-1733462887764 (Datanode Uuid 6e5f6fb1-d15a-4191-9fe6-f53a63d1617f) service to localhost/127.0.0.1:41215
2024-12-06T05:28:24,318 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:28:24,318 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data1/current/BP-2129024424-172.17.0.3-1733462887764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:24,319 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data2/current/BP-2129024424-172.17.0.3-1733462887764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:24,319 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:28:24,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5., hostname=ec46afd49254,39957,1733462890475, seqNum=2]
2024-12-06T05:28:24,324 ERROR [FSHLog-0-hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086-prefix:ec46afd49254,39957,1733462890475 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:24,324 WARN  [FSHLog-0-hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086-prefix:ec46afd49254,39957,1733462890475 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:24,324 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ec46afd49254%2C39957%2C1733462890475:(num 1733462891782) roll requested
2024-12-06T05:28:24,324 INFO  [regionserver/ec46afd49254:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C39957%2C1733462890475.1733462904324
2024-12-06T05:28:24,330 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:24,330 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:24,330 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:24,330 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:24,330 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:24,330 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462891782 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462904324
2024-12-06T05:28:24,331 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:24,331 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:24,332 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-12-06T05:28:24,332 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-12-06T05:28:24,332 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462891782
2024-12-06T05:28:24,332 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38857:38857),(127.0.0.1/127.0.0.1:40021:40021)]
2024-12-06T05:28:24,332 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462891782 is not closed yet, will try archiving it next time
2024-12-06T05:28:24,335 WARN  [IPC Server handler 2 on default port 41215 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462891782 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1009
2024-12-06T05:28:24,338 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462891782 after 5ms
2024-12-06T05:28:24,441 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:24,653 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:26,333 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:26,334 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462904324
2024-12-06T05:28:26,336 WARN  [ResponseProcessor for block BP-2129024424-172.17.0.3-1733462887764:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2129024424-172.17.0.3-1733462887764:blk_1073741838_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:26,336 WARN  [DataStreamer for file /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462904324 block BP-2129024424-172.17.0.3-1733462887764:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK], DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]) is bad.
2024-12-06T05:28:26,337 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:39586 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:40573:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39586 dst: /127.0.0.1:40573
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:26,338 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:36530 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:42285:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36530 dst: /127.0.0.1:42285
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:26,399 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1b65aae1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:26,399 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e08dd81{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:28:26,399 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:28:26,400 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@37673872{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:28:26,400 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@136e75a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.log.dir/,STOPPED}
2024-12-06T05:28:26,402 WARN  [BP-2129024424-172.17.0.3-1733462887764 heartbeating to localhost/127.0.0.1:41215 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:28:26,402 WARN  [BP-2129024424-172.17.0.3-1733462887764 heartbeating to localhost/127.0.0.1:41215 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2129024424-172.17.0.3-1733462887764 (Datanode Uuid bf1bb65e-9ed4-44da-b9e7-8b68590125bf) service to localhost/127.0.0.1:41215
2024-12-06T05:28:26,403 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:28:26,403 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data10/current/BP-2129024424-172.17.0.3-1733462887764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:26,403 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data9/current/BP-2129024424-172.17.0.3-1733462887764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:26,403 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:28:26,403 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:28:26,442 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:26,653 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:28,333 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:28,335 WARN  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]]
2024-12-06T05:28:28,335 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ec46afd49254%2C39957%2C1733462890475:(num 1733462904324) roll requested
2024-12-06T05:28:28,336 INFO  [regionserver/ec46afd49254:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C39957%2C1733462890475.1733462908335
2024-12-06T05:28:28,341 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462891782 after 4008ms
2024-12-06T05:28:28,347 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:28,347 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:28,347 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:28,347 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:28,348 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:28,348 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462904324 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462908335
2024-12-06T05:28:28,349 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39793:39793),(127.0.0.1/127.0.0.1:40021:40021)]
2024-12-06T05:28:28,349 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462891782 is not closed yet, will try archiving it next time
2024-12-06T05:28:28,349 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462904324 is not closed yet, will try archiving it next time
2024-12-06T05:28:28,350 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741838_1020 (size=2431)
2024-12-06T05:28:28,409 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-06T05:28:28,442 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:28,654 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:28,751 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462891782 is not closed yet, will try archiving it next time
2024-12-06T05:28:30,133 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741838_1020 (size=2431)
2024-12-06T05:28:30,349 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:30,414 WARN  [ResponseProcessor for block BP-2129024424-172.17.0.3-1733462887764:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2129024424-172.17.0.3-1733462887764:blk_1073741839_1021
java.io.IOException: Bad response ERROR for BP-2129024424-172.17.0.3-1733462887764:blk_1073741839_1021 from datanode DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:30,414 WARN  [DataStreamer for file /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462908335 block BP-2129024424-172.17.0.3-1733462887764:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK], DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]) is bad.
2024-12-06T05:28:30,414 WARN  [PacketResponder: BP-2129024424-172.17.0.3-1733462887764:blk_1073741839_1021, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:42285] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:30,415 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:51362 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:41703:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51362 dst: /127.0.0.1:41703
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:30,416 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:36536 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:42285:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36536 dst: /127.0.0.1:42285
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:30,442 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:30,451 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6c5f7ee0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:30,452 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@495a6aea{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:28:30,452 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:28:30,452 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ab1ed71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:28:30,452 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@65349436{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.log.dir/,STOPPED}
2024-12-06T05:28:30,455 WARN  [BP-2129024424-172.17.0.3-1733462887764 heartbeating to localhost/127.0.0.1:41215 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:28:30,455 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:28:30,455 WARN  [BP-2129024424-172.17.0.3-1733462887764 heartbeating to localhost/127.0.0.1:41215 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2129024424-172.17.0.3-1733462887764 (Datanode Uuid acf5f832-3abe-43fb-941b-af6d4505c21c) service to localhost/127.0.0.1:41215
2024-12-06T05:28:30,455 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:28:30,456 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data5/current/BP-2129024424-172.17.0.3-1733462887764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:30,456 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data6/current/BP-2129024424-172.17.0.3-1733462887764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:30,457 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:28:30,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39957 {}] regionserver.HRegion(8855): Flush requested on a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:30,467 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9e4791af9aa7cd7b9961aee9a47e3a5 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T05:28:30,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/f223a106daa44d0cba61ff99d5cf97d1 is 1080, key is row0002/info:/1733462906405/Put/seqid=0
2024-12-06T05:28:30,489 WARN  [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1023
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42285
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:30,489 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:51394 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741840_1023] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data8]'}, localName='127.0.0.1:41703', datanodeUuid='e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8', xmitsInProgress=0}:Exception transferring block BP-2129024424-172.17.0.3-1733462887764:blk_1073741840_1023 to mirror 127.0.0.1:42285
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:30,489 WARN  [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741840_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK], DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]) is bad.
2024-12-06T05:28:30,489 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:51394 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741840_1023] {}] datanode.BlockReceiver(316): Block 1073741840 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T05:28:30,490 WARN  [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741840_1023
2024-12-06T05:28:30,490 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:51394 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741840_1023] {}] datanode.DataXceiver(331): 127.0.0.1:41703:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51394 dst: /127.0.0.1:41703
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:30,492 WARN  [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]
2024-12-06T05:28:30,494 WARN  [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1024
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:30,494 WARN  [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741841_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK], DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]) is bad.
2024-12-06T05:28:30,495 WARN  [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741841_1024
2024-12-06T05:28:30,495 WARN  [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]
2024-12-06T05:28:30,496 WARN  [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1025
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:30,496 WARN  [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK], DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]) is bad.
2024-12-06T05:28:30,496 WARN  [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741842_1025
2024-12-06T05:28:30,497 WARN  [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]
2024-12-06T05:28:30,499 WARN  [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40573
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:30,499 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:51404 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741843_1026] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data8]'}, localName='127.0.0.1:41703', datanodeUuid='e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8', xmitsInProgress=0}:Exception transferring block BP-2129024424-172.17.0.3-1733462887764:blk_1073741843_1026 to mirror 127.0.0.1:40573
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:30,499 WARN  [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK], DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]) is bad.
2024-12-06T05:28:30,499 WARN  [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741843_1026
2024-12-06T05:28:30,499 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:51404 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741843_1026] {}] datanode.BlockReceiver(316): Block 1073741843 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T05:28:30,499 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:51404 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741843_1026] {}] datanode.DataXceiver(331): 127.0.0.1:41703:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51404 dst: /127.0.0.1:41703
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:30,500 WARN  [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]
2024-12-06T05:28:30,500 WARN  [IPC Server handler 1 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-06T05:28:30,500 WARN  [IPC Server handler 1 on default port 41215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-06T05:28:30,500 WARN  [IPC Server handler 1 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-06T05:28:30,503 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741844_1027 (size=10347)
2024-12-06T05:28:30,654 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:30,906 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/f223a106daa44d0cba61ff99d5cf97d1
2024-12-06T05:28:30,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/f223a106daa44d0cba61ff99d5cf97d1 as hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/f223a106daa44d0cba61ff99d5cf97d1
2024-12-06T05:28:30,925 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/f223a106daa44d0cba61ff99d5cf97d1, entries=5, sequenceid=11, filesize=10.1 K
2024-12-06T05:28:30,926 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for a9e4791af9aa7cd7b9961aee9a47e3a5 in 459ms, sequenceid=11, compaction requested=false
2024-12-06T05:28:30,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9e4791af9aa7cd7b9961aee9a47e3a5:

2024-12-06T05:28:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39957 {}] regionserver.HRegion(8855): Flush requested on a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:31,102 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9e4791af9aa7cd7b9961aee9a47e3a5 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-12-06T05:28:31,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/705153fdc66c4999b49772d17d1beb95 is 1080, key is row0007/info:/1733462910469/Put/seqid=0
2024-12-06T05:28:31,112 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:31,113 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK], DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]) is bad.
2024-12-06T05:28:31,113 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741845_1028
2024-12-06T05:28:31,113 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]
2024-12-06T05:28:31,116 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42285
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:31,116 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:51436 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741846_1029] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data8]'}, localName='127.0.0.1:41703', datanodeUuid='e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8', xmitsInProgress=0}:Exception transferring block BP-2129024424-172.17.0.3-1733462887764:blk_1073741846_1029 to mirror 127.0.0.1:42285
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:31,116 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK], DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]) is bad.
2024-12-06T05:28:31,116 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741846_1029
2024-12-06T05:28:31,116 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:51436 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741846_1029] {}] datanode.BlockReceiver(316): Block 1073741846 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T05:28:31,116 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:51436 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741846_1029] {}] datanode.DataXceiver(331): 127.0.0.1:41703:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51436 dst: /127.0.0.1:41703
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:31,117 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]
2024-12-06T05:28:31,119 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:31,119 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK], DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]) is bad.
2024-12-06T05:28:31,119 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741847_1030
2024-12-06T05:28:31,120 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]
2024-12-06T05:28:31,121 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:31,121 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK], DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]) is bad.
2024-12-06T05:28:31,121 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741848_1031
2024-12-06T05:28:31,122 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]
2024-12-06T05:28:31,122 WARN  [IPC Server handler 1 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-06T05:28:31,122 WARN  [IPC Server handler 1 on default port 41215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-06T05:28:31,123 WARN  [IPC Server handler 1 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-06T05:28:31,127 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741849_1032 (size=12506)
2024-12-06T05:28:31,528 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/705153fdc66c4999b49772d17d1beb95
2024-12-06T05:28:31,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/705153fdc66c4999b49772d17d1beb95 as hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/705153fdc66c4999b49772d17d1beb95
2024-12-06T05:28:31,541 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/705153fdc66c4999b49772d17d1beb95, entries=7, sequenceid=24, filesize=12.2 K
2024-12-06T05:28:31,542 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for a9e4791af9aa7cd7b9961aee9a47e3a5 in 440ms, sequenceid=24, compaction requested=false
2024-12-06T05:28:31,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9e4791af9aa7cd7b9961aee9a47e3a5:

2024-12-06T05:28:31,542 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-12-06T05:28:31,542 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:28:31,542 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/705153fdc66c4999b49772d17d1beb95 because midkey is the same as first or last row
2024-12-06T05:28:32,350 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:32,350 WARN  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]
2024-12-06T05:28:32,350 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ec46afd49254%2C39957%2C1733462890475:(num 1733462908335) roll requested
2024-12-06T05:28:32,351 INFO  [regionserver/ec46afd49254:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C39957%2C1733462890475.1733462912351
2024-12-06T05:28:32,356 WARN  [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41475
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:32,356 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59026 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741850_1033] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data8]'}, localName='127.0.0.1:41703', datanodeUuid='e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8', xmitsInProgress=0}:Exception transferring block BP-2129024424-172.17.0.3-1733462887764:blk_1073741850_1033 to mirror 127.0.0.1:41475
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:32,356 WARN  [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK], DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]) is bad.
2024-12-06T05:28:32,356 WARN  [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741850_1033
2024-12-06T05:28:32,356 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59026 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741850_1033] {}] datanode.BlockReceiver(316): Block 1073741850 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-06T05:28:32,356 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59026 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741850_1033] {}] datanode.DataXceiver(331): 127.0.0.1:41703:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59026 dst: /127.0.0.1:41703
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:32,357 WARN  [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]
2024-12-06T05:28:32,359 WARN  [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:32,359 WARN  [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK], DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]) is bad.
2024-12-06T05:28:32,359 WARN  [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741851_1034
2024-12-06T05:28:32,360 WARN  [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]
2024-12-06T05:28:32,361 WARN  [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:32,362 WARN  [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK], DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]) is bad.
2024-12-06T05:28:32,362 WARN  [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741852_1035
2024-12-06T05:28:32,362 WARN  [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]
2024-12-06T05:28:32,365 WARN  [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44389
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:32,365 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59032 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741853_1036] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data8]'}, localName='127.0.0.1:41703', datanodeUuid='e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8', xmitsInProgress=0}:Exception transferring block BP-2129024424-172.17.0.3-1733462887764:blk_1073741853_1036 to mirror 127.0.0.1:44389
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:32,366 WARN  [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK], DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]) is bad.
2024-12-06T05:28:32,366 WARN  [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741853_1036
2024-12-06T05:28:32,366 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59032 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741853_1036] {}] datanode.BlockReceiver(316): Block 1073741853 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-06T05:28:32,366 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59032 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741853_1036] {}] datanode.DataXceiver(331): 127.0.0.1:41703:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59032 dst: /127.0.0.1:41703
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:32,366 WARN  [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]
2024-12-06T05:28:32,367 WARN  [IPC Server handler 2 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-06T05:28:32,367 WARN  [IPC Server handler 2 on default port 41215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-06T05:28:32,368 WARN  [IPC Server handler 2 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-06T05:28:32,376 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:32,376 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:32,376 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:32,377 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:32,377 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:32,377 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462908335 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462912351
2024-12-06T05:28:32,378 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39793:39793)]
2024-12-06T05:28:32,378 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462891782 is not closed yet, will try archiving it next time
2024-12-06T05:28:32,378 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462908335 is not closed yet, will try archiving it next time
2024-12-06T05:28:32,378 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462904324 to hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/oldWALs/ec46afd49254%2C39957%2C1733462890475.1733462904324
2024-12-06T05:28:32,379 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741839_1022 (size=25992)
2024-12-06T05:28:32,443 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:32,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39957 {}] regionserver.HRegion(8855): Flush requested on a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:32,531 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9e4791af9aa7cd7b9961aee9a47e3a5 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-06T05:28:32,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/b78e83e1f0d841fcb2ebdc994da2ab5f is 1079, key is tmprow/info:/1733462912529/Put/seqid=0
2024-12-06T05:28:32,537 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:32,537 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK], DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]) is bad.
2024-12-06T05:28:32,537 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741855_1038
2024-12-06T05:28:32,538 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]
2024-12-06T05:28:32,539 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:32,539 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK], DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]) is bad.
2024-12-06T05:28:32,539 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741856_1039
2024-12-06T05:28:32,540 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]
2024-12-06T05:28:32,541 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:32,542 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK], DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]) is bad.
2024-12-06T05:28:32,542 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741857_1040
2024-12-06T05:28:32,545 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]
2024-12-06T05:28:32,546 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:32,547 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK], DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]) is bad.
2024-12-06T05:28:32,547 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741858_1041
2024-12-06T05:28:32,548 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]
2024-12-06T05:28:32,549 WARN  [IPC Server handler 4 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-06T05:28:32,549 WARN  [IPC Server handler 4 on default port 41215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-06T05:28:32,549 WARN  [IPC Server handler 4 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-06T05:28:32,557 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741859_1042 (size=6027)
2024-12-06T05:28:32,558 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/b78e83e1f0d841fcb2ebdc994da2ab5f
2024-12-06T05:28:32,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/b78e83e1f0d841fcb2ebdc994da2ab5f as hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/b78e83e1f0d841fcb2ebdc994da2ab5f
2024-12-06T05:28:32,581 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/b78e83e1f0d841fcb2ebdc994da2ab5f, entries=1, sequenceid=34, filesize=5.9 K
2024-12-06T05:28:32,583 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=0 B/0 for a9e4791af9aa7cd7b9961aee9a47e3a5 in 52ms, sequenceid=34, compaction requested=true
2024-12-06T05:28:32,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9e4791af9aa7cd7b9961aee9a47e3a5:

2024-12-06T05:28:32,584 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-12-06T05:28:32,584 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:28:32,584 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/705153fdc66c4999b49772d17d1beb95 because midkey is the same as first or last row
2024-12-06T05:28:32,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9e4791af9aa7cd7b9961aee9a47e3a5:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T05:28:32,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:28:32,588 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T05:28:32,590 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T05:28:32,590 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.HStore(1541): a9e4791af9aa7cd7b9961aee9a47e3a5/info is initiating minor compaction (all files)
2024-12-06T05:28:32,590 INFO  [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9e4791af9aa7cd7b9961aee9a47e3a5/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.
2024-12-06T05:28:32,590 INFO  [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/f223a106daa44d0cba61ff99d5cf97d1, hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/705153fdc66c4999b49772d17d1beb95, hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/b78e83e1f0d841fcb2ebdc994da2ab5f] into tmpdir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp, totalSize=28.2 K
2024-12-06T05:28:32,591 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f223a106daa44d0cba61ff99d5cf97d1, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733462906405
2024-12-06T05:28:32,591 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 705153fdc66c4999b49772d17d1beb95, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1733462910469
2024-12-06T05:28:32,592 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b78e83e1f0d841fcb2ebdc994da2ab5f, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733462912529
2024-12-06T05:28:32,607 INFO  [RS:0;ec46afd49254:39957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9e4791af9aa7cd7b9961aee9a47e3a5#info#compaction#21 average throughput is 4.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T05:28:32,608 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/c6cfb60cfdd149278bbd0f1ce46d9c97 is 1080, key is row0002/info:/1733462906405/Put/seqid=0
2024-12-06T05:28:32,613 WARN  [Thread-945 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42285
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:32,613 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59064 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741860_1043] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data8]'}, localName='127.0.0.1:41703', datanodeUuid='e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8', xmitsInProgress=0}:Exception transferring block BP-2129024424-172.17.0.3-1733462887764:blk_1073741860_1043 to mirror 127.0.0.1:42285
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:32,614 WARN  [Thread-945 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK], DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]) is bad.
2024-12-06T05:28:32,614 WARN  [Thread-945 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741860_1043
2024-12-06T05:28:32,614 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59064 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741860_1043] {}] datanode.BlockReceiver(316): Block 1073741860 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T05:28:32,614 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59064 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741860_1043] {}] datanode.DataXceiver(331): 127.0.0.1:41703:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59064 dst: /127.0.0.1:41703
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:32,615 WARN  [Thread-945 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]
2024-12-06T05:28:32,620 WARN  [Thread-945 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:32,620 WARN  [Thread-945 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK], DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]) is bad.
2024-12-06T05:28:32,620 WARN  [Thread-945 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741861_1044
2024-12-06T05:28:32,621 WARN  [Thread-945 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]
2024-12-06T05:28:32,622 WARN  [Thread-945 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:32,622 WARN  [Thread-945 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK], DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]) is bad.
2024-12-06T05:28:32,622 WARN  [Thread-945 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741862_1045
2024-12-06T05:28:32,623 WARN  [Thread-945 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]
2024-12-06T05:28:32,624 WARN  [Thread-945 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:32,625 WARN  [Thread-945 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK], DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]) is bad.
2024-12-06T05:28:32,625 WARN  [Thread-945 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741863_1046
2024-12-06T05:28:32,625 WARN  [Thread-945 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]
2024-12-06T05:28:32,626 WARN  [IPC Server handler 4 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-06T05:28:32,626 WARN  [IPC Server handler 4 on default port 41215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-06T05:28:32,626 WARN  [IPC Server handler 4 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-06T05:28:32,633 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741864_1047 (size=17994)
2024-12-06T05:28:32,642 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/c6cfb60cfdd149278bbd0f1ce46d9c97 as hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/c6cfb60cfdd149278bbd0f1ce46d9c97
2024-12-06T05:28:32,653 INFO  [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9e4791af9aa7cd7b9961aee9a47e3a5/info of a9e4791af9aa7cd7b9961aee9a47e3a5 into c6cfb60cfdd149278bbd0f1ce46d9c97(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T05:28:32,653 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9e4791af9aa7cd7b9961aee9a47e3a5:

2024-12-06T05:28:32,653 INFO  [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5., storeName=a9e4791af9aa7cd7b9961aee9a47e3a5/info, priority=13, startTime=1733462912584; duration=0sec
2024-12-06T05:28:32,653 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-06T05:28:32,653 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:28:32,654 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/c6cfb60cfdd149278bbd0f1ce46d9c97 because midkey is the same as first or last row
2024-12-06T05:28:32,654 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-06T05:28:32,654 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:28:32,654 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/c6cfb60cfdd149278bbd0f1ce46d9c97 because midkey is the same as first or last row
2024-12-06T05:28:32,654 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-06T05:28:32,654 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:28:32,654 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/c6cfb60cfdd149278bbd0f1ce46d9c97 because midkey is the same as first or last row
2024-12-06T05:28:32,654 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:28:32,655 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9e4791af9aa7cd7b9961aee9a47e3a5:info
2024-12-06T05:28:32,655 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:32,780 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462891782 is not closed yet, will try archiving it next time
2024-12-06T05:28:32,780 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462908335 to hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/oldWALs/ec46afd49254%2C39957%2C1733462890475.1733462908335
2024-12-06T05:28:33,220 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@49b73ca3[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41703, datanodeUuid=e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8, infoPort=39793, infoSecurePort=0, ipcPort=44537, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764):Failed to transfer BP-2129024424-172.17.0.3-1733462887764:blk_1073741844_1027 to 127.0.0.1:44389 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:33,220 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4770dbe5[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41703, datanodeUuid=e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8, infoPort=39793, infoSecurePort=0, ipcPort=44537, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764):Failed to transfer BP-2129024424-172.17.0.3-1733462887764:blk_1073741849_1032 to 127.0.0.1:44389 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39957 {}] regionserver.HRegion(8855): Flush requested on a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:33,956 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9e4791af9aa7cd7b9961aee9a47e3a5 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-06T05:28:33,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/01d4cb4831f44595ad6146b725073a95 is 1079, key is tmprow/info:/1733462913955/Put/seqid=0
2024-12-06T05:28:33,964 WARN  [Thread-954 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42285
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:33,964 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59088 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741865_1048] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data8]'}, localName='127.0.0.1:41703', datanodeUuid='e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8', xmitsInProgress=0}:Exception transferring block BP-2129024424-172.17.0.3-1733462887764:blk_1073741865_1048 to mirror 127.0.0.1:42285
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:33,964 WARN  [Thread-954 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK], DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]) is bad.
2024-12-06T05:28:33,964 WARN  [Thread-954 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741865_1048
2024-12-06T05:28:33,964 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59088 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741865_1048] {}] datanode.BlockReceiver(316): Block 1073741865 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T05:28:33,964 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59088 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741865_1048] {}] datanode.DataXceiver(331): 127.0.0.1:41703:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59088 dst: /127.0.0.1:41703
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:33,965 WARN  [Thread-954 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]
2024-12-06T05:28:33,966 WARN  [Thread-954 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:33,966 WARN  [Thread-954 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK], DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]) is bad.
2024-12-06T05:28:33,966 WARN  [Thread-954 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741866_1049
2024-12-06T05:28:33,966 WARN  [Thread-954 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]
2024-12-06T05:28:33,968 WARN  [Thread-954 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:33,968 WARN  [Thread-954 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK], DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]) is bad.
2024-12-06T05:28:33,968 WARN  [Thread-954 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741867_1050
2024-12-06T05:28:33,968 WARN  [Thread-954 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]
2024-12-06T05:28:33,973 WARN  [Thread-954 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41475
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:33,973 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59098 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741868_1051] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data8]'}, localName='127.0.0.1:41703', datanodeUuid='e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8', xmitsInProgress=0}:Exception transferring block BP-2129024424-172.17.0.3-1733462887764:blk_1073741868_1051 to mirror 127.0.0.1:41475
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:33,973 WARN  [Thread-954 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK], DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]) is bad.
2024-12-06T05:28:33,973 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59098 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741868_1051] {}] datanode.BlockReceiver(316): Block 1073741868 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T05:28:33,973 WARN  [Thread-954 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741868_1051
2024-12-06T05:28:33,973 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59098 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741868_1051] {}] datanode.DataXceiver(331): 127.0.0.1:41703:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59098 dst: /127.0.0.1:41703
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:33,974 WARN  [Thread-954 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]
2024-12-06T05:28:33,975 WARN  [IPC Server handler 3 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-06T05:28:33,975 WARN  [IPC Server handler 3 on default port 41215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-06T05:28:33,975 WARN  [IPC Server handler 3 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-06T05:28:33,979 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741869_1052 (size=6027)
2024-12-06T05:28:34,219 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4770dbe5[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41703, datanodeUuid=e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8, infoPort=39793, infoSecurePort=0, ipcPort=44537, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764):Failed to transfer BP-2129024424-172.17.0.3-1733462887764:blk_1073741839_1022 to 127.0.0.1:42285 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:34,378 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:34,378 WARN  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]
2024-12-06T05:28:34,378 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ec46afd49254%2C39957%2C1733462890475:(num 1733462912351) roll requested
2024-12-06T05:28:34,379 INFO  [regionserver/ec46afd49254:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C39957%2C1733462890475.1733462914379
2024-12-06T05:28:34,380 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/01d4cb4831f44595ad6146b725073a95
2024-12-06T05:28:34,382 WARN  [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:34,382 WARN  [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK], DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]) is bad.
2024-12-06T05:28:34,382 WARN  [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741870_1053
2024-12-06T05:28:34,383 WARN  [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]
2024-12-06T05:28:34,385 WARN  [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:34,385 WARN  [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK], DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]) is bad.
2024-12-06T05:28:34,386 WARN  [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741871_1054
2024-12-06T05:28:34,387 WARN  [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]
2024-12-06T05:28:34,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/01d4cb4831f44595ad6146b725073a95 as hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/01d4cb4831f44595ad6146b725073a95
2024-12-06T05:28:34,391 WARN  [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:34,391 WARN  [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK], DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]) is bad.
2024-12-06T05:28:34,391 WARN  [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741872_1055
2024-12-06T05:28:34,391 WARN  [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]
2024-12-06T05:28:34,393 WARN  [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:34,393 WARN  [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK], DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]) is bad.
2024-12-06T05:28:34,393 WARN  [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741873_1056
2024-12-06T05:28:34,394 WARN  [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]
2024-12-06T05:28:34,396 WARN  [IPC Server handler 3 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-06T05:28:34,396 WARN  [IPC Server handler 3 on default port 41215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-06T05:28:34,396 WARN  [IPC Server handler 3 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-06T05:28:34,399 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/01d4cb4831f44595ad6146b725073a95, entries=1, sequenceid=45, filesize=5.9 K
2024-12-06T05:28:34,400 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:34,400 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for a9e4791af9aa7cd7b9961aee9a47e3a5 in 444ms, sequenceid=45, compaction requested=false
2024-12-06T05:28:34,400 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:34,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9e4791af9aa7cd7b9961aee9a47e3a5:

2024-12-06T05:28:34,401 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:34,401 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:34,401 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-12-06T05:28:34,401 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:28:34,401 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:34,401 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/c6cfb60cfdd149278bbd0f1ce46d9c97 because midkey is the same as first or last row
2024-12-06T05:28:34,401 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462912351 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462914379
2024-12-06T05:28:34,403 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741854_1037 (size=13591)
2024-12-06T05:28:34,404 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462891782 is not closed yet, will try archiving it next time
2024-12-06T05:28:34,405 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39793:39793)]
2024-12-06T05:28:34,405 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462891782 is not closed yet, will try archiving it next time
2024-12-06T05:28:34,443 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:34,655 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:35,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39957 {}] regionserver.HRegion(8855): Flush requested on a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:35,378 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9e4791af9aa7cd7b9961aee9a47e3a5 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-06T05:28:35,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/bfe047c8a1d74ab0910c902e2dfeee49 is 1079, key is tmprow/info:/1733462915376/Put/seqid=0
2024-12-06T05:28:35,384 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:35,384 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK], DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]) is bad.
2024-12-06T05:28:35,384 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741875_1058
2024-12-06T05:28:35,385 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]
2024-12-06T05:28:35,386 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:35,386 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK], DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]) is bad.
2024-12-06T05:28:35,386 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741876_1059
2024-12-06T05:28:35,386 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]
2024-12-06T05:28:35,387 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:35,387 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK], DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]) is bad.
2024-12-06T05:28:35,387 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741877_1060
2024-12-06T05:28:35,388 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]
2024-12-06T05:28:35,389 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:35,389 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK], DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]) is bad.
2024-12-06T05:28:35,389 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741878_1061
2024-12-06T05:28:35,390 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]
2024-12-06T05:28:35,390 WARN  [IPC Server handler 2 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-06T05:28:35,390 WARN  [IPC Server handler 2 on default port 41215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-06T05:28:35,390 WARN  [IPC Server handler 2 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-06T05:28:35,393 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741879_1062 (size=6027)
2024-12-06T05:28:35,795 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/bfe047c8a1d74ab0910c902e2dfeee49
2024-12-06T05:28:35,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/bfe047c8a1d74ab0910c902e2dfeee49 as hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/bfe047c8a1d74ab0910c902e2dfeee49
2024-12-06T05:28:35,807 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/bfe047c8a1d74ab0910c902e2dfeee49, entries=1, sequenceid=55, filesize=5.9 K
2024-12-06T05:28:35,808 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for a9e4791af9aa7cd7b9961aee9a47e3a5 in 431ms, sequenceid=55, compaction requested=true
2024-12-06T05:28:35,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9e4791af9aa7cd7b9961aee9a47e3a5:

2024-12-06T05:28:35,808 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-12-06T05:28:35,808 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:28:35,808 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/c6cfb60cfdd149278bbd0f1ce46d9c97 because midkey is the same as first or last row
2024-12-06T05:28:35,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9e4791af9aa7cd7b9961aee9a47e3a5:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T05:28:35,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:28:35,808 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T05:28:35,809 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T05:28:35,810 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.HStore(1541): a9e4791af9aa7cd7b9961aee9a47e3a5/info is initiating minor compaction (all files)
2024-12-06T05:28:35,810 INFO  [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9e4791af9aa7cd7b9961aee9a47e3a5/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.
2024-12-06T05:28:35,810 INFO  [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/c6cfb60cfdd149278bbd0f1ce46d9c97, hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/01d4cb4831f44595ad6146b725073a95, hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/bfe047c8a1d74ab0910c902e2dfeee49] into tmpdir=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp, totalSize=29.3 K
2024-12-06T05:28:35,810 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6cfb60cfdd149278bbd0f1ce46d9c97, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733462906405
2024-12-06T05:28:35,811 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01d4cb4831f44595ad6146b725073a95, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733462913955
2024-12-06T05:28:35,811 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfe047c8a1d74ab0910c902e2dfeee49, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733462915376
2024-12-06T05:28:35,826 INFO  [RS:0;ec46afd49254:39957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9e4791af9aa7cd7b9961aee9a47e3a5#info#compaction#24 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T05:28:35,827 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/9dfd627ed399444a9a05bb0af7c086a4 is 1080, key is row0002/info:/1733462906405/Put/seqid=0
2024-12-06T05:28:35,829 WARN  [Thread-968 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:35,829 WARN  [Thread-968 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK], DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]) is bad.
2024-12-06T05:28:35,829 WARN  [Thread-968 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741880_1063
2024-12-06T05:28:35,829 WARN  [Thread-968 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]
2024-12-06T05:28:35,830 WARN  [Thread-968 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:35,830 WARN  [Thread-968 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK], DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]) is bad.
2024-12-06T05:28:35,831 WARN  [Thread-968 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741881_1064
2024-12-06T05:28:35,831 WARN  [Thread-968 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44389,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]
2024-12-06T05:28:35,832 WARN  [Thread-968 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:35,832 WARN  [Thread-968 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK], DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]) is bad.
2024-12-06T05:28:35,833 WARN  [Thread-968 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741882_1065
2024-12-06T05:28:35,833 WARN  [Thread-968 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]
2024-12-06T05:28:35,836 WARN  [Thread-968 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41475
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:35,836 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59124 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741883_1066] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data8]'}, localName='127.0.0.1:41703', datanodeUuid='e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8', xmitsInProgress=0}:Exception transferring block BP-2129024424-172.17.0.3-1733462887764:blk_1073741883_1066 to mirror 127.0.0.1:41475
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:35,836 WARN  [Thread-968 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK], DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]) is bad.
2024-12-06T05:28:35,836 WARN  [Thread-968 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741883_1066
2024-12-06T05:28:35,836 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59124 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741883_1066] {}] datanode.BlockReceiver(316): Block 1073741883 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T05:28:35,836 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:59124 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741883_1066] {}] datanode.DataXceiver(331): 127.0.0.1:41703:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59124 dst: /127.0.0.1:41703
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:35,837 WARN  [Thread-968 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]
2024-12-06T05:28:35,838 WARN  [IPC Server handler 2 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-06T05:28:35,838 WARN  [IPC Server handler 2 on default port 41215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-06T05:28:35,838 WARN  [IPC Server handler 2 on default port 41215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-06T05:28:35,841 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741884_1067 (size=18097)
2024-12-06T05:28:36,220 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4770dbe5[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41703, datanodeUuid=e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8, infoPort=39793, infoSecurePort=0, ipcPort=44537, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764):Failed to transfer BP-2129024424-172.17.0.3-1733462887764:blk_1073741859_1042 to 127.0.0.1:40573 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:36,220 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@49b73ca3[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41703, datanodeUuid=e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8, infoPort=39793, infoSecurePort=0, ipcPort=44537, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764):Failed to transfer BP-2129024424-172.17.0.3-1733462887764:blk_1073741864_1047 to 127.0.0.1:44389 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:36,255 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/9dfd627ed399444a9a05bb0af7c086a4 as hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/9dfd627ed399444a9a05bb0af7c086a4
2024-12-06T05:28:36,263 INFO  [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9e4791af9aa7cd7b9961aee9a47e3a5/info of a9e4791af9aa7cd7b9961aee9a47e3a5 into 9dfd627ed399444a9a05bb0af7c086a4(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T05:28:36,263 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9e4791af9aa7cd7b9961aee9a47e3a5:

2024-12-06T05:28:36,263 INFO  [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5., storeName=a9e4791af9aa7cd7b9961aee9a47e3a5/info, priority=13, startTime=1733462915808; duration=0sec
2024-12-06T05:28:36,263 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-12-06T05:28:36,263 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:28:36,263 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/9dfd627ed399444a9a05bb0af7c086a4 because midkey is the same as first or last row
2024-12-06T05:28:36,263 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-12-06T05:28:36,263 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:28:36,264 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/9dfd627ed399444a9a05bb0af7c086a4 because midkey is the same as first or last row
2024-12-06T05:28:36,264 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-12-06T05:28:36,264 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:28:36,264 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/9dfd627ed399444a9a05bb0af7c086a4 because midkey is the same as first or last row
2024-12-06T05:28:36,264 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:28:36,264 DEBUG [RS:0;ec46afd49254:39957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9e4791af9aa7cd7b9961aee9a47e3a5:info
2024-12-06T05:28:36,405 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:36,405 WARN  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-12-06T05:28:36,443 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:36,608 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:28:36,611 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:28:36,612 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:28:36,612 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:28:36,612 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T05:28:36,613 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54f27916{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:28:36,614 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@557202e0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:28:36,656 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:36,708 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@15b530e2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/java.io.tmpdir/jetty-localhost-44195-hadoop-hdfs-3_4_1-tests_jar-_-any-7212905443893919449/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:36,709 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7f6709ee{HTTP/1.1, (http/1.1)}{localhost:44195}
2024-12-06T05:28:36,709 INFO  [Time-limited test {}] server.Server(415): Started @136320ms
2024-12-06T05:28:36,710 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:28:37,219 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4770dbe5[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41703, datanodeUuid=e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8, infoPort=39793, infoSecurePort=0, ipcPort=44537, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764):Failed to transfer BP-2129024424-172.17.0.3-1733462887764:blk_1073741854_1037 to 127.0.0.1:42285 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:37,219 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@49b73ca3[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41703, datanodeUuid=e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8, infoPort=39793, infoSecurePort=0, ipcPort=44537, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764):Failed to transfer BP-2129024424-172.17.0.3-1733462887764:blk_1073741869_1052 to 127.0.0.1:42285 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:37,369 WARN  [Thread-987 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:28:37,380 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xae9cb7927ab21104 with lease ID 0xba9d9ca3b0b8bed2: from storage DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96 node DatanodeRegistration(127.0.0.1:38435, datanodeUuid=e97cdf8a-79a9-4085-ba36-a94e2f6ff519, infoPort=36195, infoSecurePort=0, ipcPort=45427, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:28:37,380 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xae9cb7927ab21104 with lease ID 0xba9d9ca3b0b8bed2: from storage DS-86c9fffd-f083-42a7-b3d5-009b11562dea node DatanodeRegistration(127.0.0.1:38435, datanodeUuid=e97cdf8a-79a9-4085-ba36-a94e2f6ff519, infoPort=36195, infoSecurePort=0, ipcPort=45427, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:28:38,406 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:38,444 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:38,656 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:39,220 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@49b73ca3[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41703, datanodeUuid=e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8, infoPort=39793, infoSecurePort=0, ipcPort=44537, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764):Failed to transfer BP-2129024424-172.17.0.3-1733462887764:blk_1073741884_1067 to 127.0.0.1:42285 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:39,220 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4770dbe5[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41703, datanodeUuid=e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8, infoPort=39793, infoSecurePort=0, ipcPort=44537, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764):Failed to transfer BP-2129024424-172.17.0.3-1733462887764:blk_1073741879_1062 to 127.0.0.1:42285 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:40,086 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T05:28:40,406 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:40,444 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:40,656 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:41,502 ERROR [FSHLog-0-hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData-prefix:ec46afd49254,34167,1733462890110 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:41,502 WARN  [FSHLog-0-hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData-prefix:ec46afd49254,34167,1733462890110 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:41,502 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog ec46afd49254%2C34167%2C1733462890110:(num 1733462890673) roll requested
2024-12-06T05:28:41,503 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C34167%2C1733462890110.1733462921502
2024-12-06T05:28:41,506 WARN  [Thread-1007 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:41,507 WARN  [Thread-1007 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK], DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]) is bad.
2024-12-06T05:28:41,507 WARN  [Thread-1007 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741885_1068
2024-12-06T05:28:41,508 WARN  [Thread-1007 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]
2024-12-06T05:28:41,512 WARN  [Thread-1007 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1069
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:41,512 WARN  [Thread-1007 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741886_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK], DatanodeInfoWithStorage[127.0.0.1:38435,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]) is bad.
2024-12-06T05:28:41,512 WARN  [Thread-1007 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741886_1069
2024-12-06T05:28:41,513 WARN  [Thread-1007 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]
2024-12-06T05:28:41,519 WARN  [Thread-1007 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1070
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40573
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:41,519 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1189303528_22 at /127.0.0.1:59148 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741887_1070] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data8]'}, localName='127.0.0.1:41703', datanodeUuid='e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8', xmitsInProgress=0}:Exception transferring block BP-2129024424-172.17.0.3-1733462887764:blk_1073741887_1070 to mirror 127.0.0.1:40573
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:41,519 WARN  [Thread-1007 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741887_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK], DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]) is bad.
2024-12-06T05:28:41,519 WARN  [Thread-1007 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741887_1070
2024-12-06T05:28:41,519 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1189303528_22 at /127.0.0.1:59148 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741887_1070] {}] datanode.BlockReceiver(316): Block 1073741887 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-06T05:28:41,519 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1189303528_22 at /127.0.0.1:59148 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741887_1070] {}] datanode.DataXceiver(331): 127.0.0.1:41703:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59148 dst: /127.0.0.1:41703
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:41,520 WARN  [Thread-1007 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]
2024-12-06T05:28:41,525 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:41,525 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:41,525 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:41,526 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:41,526 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:41,526 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/WALs/ec46afd49254,34167,1733462890110/ec46afd49254%2C34167%2C1733462890110.1733462890673 with entries=54, filesize=26.67 KB; new WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/WALs/ec46afd49254,34167,1733462890110/ec46afd49254%2C34167%2C1733462890110.1733462921502
2024-12-06T05:28:41,526 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:41,527 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:41,527 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/WALs/ec46afd49254,34167,1733462890110/ec46afd49254%2C34167%2C1733462890110.1733462890673
2024-12-06T05:28:41,527 WARN  [IPC Server handler 0 on default port 41215 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/WALs/ec46afd49254,34167,1733462890110/ec46afd49254%2C34167%2C1733462890110.1733462890673 has not been closed. Lease recovery is in progress. RecoveryId = 1072 for block blk_1073741830_1006
2024-12-06T05:28:41,527 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36195:36195),(127.0.0.1/127.0.0.1:39793:39793)]
2024-12-06T05:28:41,527 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/WALs/ec46afd49254,34167,1733462890110/ec46afd49254%2C34167%2C1733462890110.1733462890673 is not closed yet, will try archiving it next time
2024-12-06T05:28:41,527 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/WALs/ec46afd49254,34167,1733462890110/ec46afd49254%2C34167%2C1733462890110.1733462890673 after 0ms
2024-12-06T05:28:42,407 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:42,445 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:44,407 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:44,445 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:45,529 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/WALs/ec46afd49254,34167,1733462890110/ec46afd49254%2C34167%2C1733462890110.1733462890673 after 4002ms
2024-12-06T05:28:46,408 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:46,445 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:47,393 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@84f0672 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-2129024424-172.17.0.3-1733462887764:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:41475,null,null])
java.net.ConnectException: Call From ec46afd49254/172.17.0.3 to localhost:46841 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-06T05:28:47,408 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38435 is added to blk_1073741833_1019 (size=455)
2024-12-06T05:28:48,365 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462891782 to hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/oldWALs/ec46afd49254%2C39957%2C1733462890475.1733462891782
2024-12-06T05:28:48,366 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462912351 to hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/oldWALs/ec46afd49254%2C39957%2C1733462890475.1733462912351
2024-12-06T05:28:48,408 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:48,446 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,143 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C39957%2C1733462890475.1733462930143
2024-12-06T05:28:50,146 WARN  [Thread-1017 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741889_1073
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,146 WARN  [Thread-1017 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741889_1073 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK], DatanodeInfoWithStorage[127.0.0.1:38435,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]) is bad.
2024-12-06T05:28:50,146 WARN  [Thread-1017 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741889_1073
2024-12-06T05:28:50,147 WARN  [Thread-1017 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]
2024-12-06T05:28:50,149 WARN  [Thread-1017 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1074
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40573
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,149 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1189303528_22 at /127.0.0.1:44732 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741890_1074] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data8]'}, localName='127.0.0.1:41703', datanodeUuid='e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8', xmitsInProgress=0}:Exception transferring block BP-2129024424-172.17.0.3-1733462887764:blk_1073741890_1074 to mirror 127.0.0.1:40573
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:50,149 WARN  [Thread-1017 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741890_1074 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK], DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]) is bad.
2024-12-06T05:28:50,149 WARN  [Thread-1017 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741890_1074
2024-12-06T05:28:50,149 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1189303528_22 at /127.0.0.1:44732 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741890_1074] {}] datanode.BlockReceiver(316): Block 1073741890 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-06T05:28:50,149 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1189303528_22 at /127.0.0.1:44732 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741890_1074] {}] datanode.DataXceiver(331): 127.0.0.1:41703:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:44732 dst: /127.0.0.1:41703
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:50,150 WARN  [Thread-1017 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]
2024-12-06T05:28:50,154 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:50,154 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:50,154 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:50,155 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:50,155 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:50,155 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462914379 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462930143
2024-12-06T05:28:50,156 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39793:39793),(127.0.0.1/127.0.0.1:36195:36195)]
2024-12-06T05:28:50,156 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462914379 is not closed yet, will try archiving it next time
2024-12-06T05:28:50,156 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741874_1057 (size=12911)
2024-12-06T05:28:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39957 {}] regionserver.HRegion(8855): Flush requested on a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:50,163 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9e4791af9aa7cd7b9961aee9a47e3a5 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-06T05:28:50,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/eb98167d238b4108810871cb8d0e9d25 is 1080, key is row0013/info:/1733462930157/Put/seqid=0
2024-12-06T05:28:50,170 WARN  [Thread-1024 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741892_1076
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,171 WARN  [Thread-1024 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741892_1076 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK], DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]) is bad.
2024-12-06T05:28:50,171 WARN  [Thread-1024 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741892_1076
2024-12-06T05:28:50,171 WARN  [Thread-1024 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]
2024-12-06T05:28:50,173 WARN  [Thread-1024 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741893_1077
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,173 WARN  [Thread-1024 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741893_1077 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK], DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]) is bad.
2024-12-06T05:28:50,173 WARN  [Thread-1024 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741893_1077
2024-12-06T05:28:50,174 WARN  [Thread-1024 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]
2024-12-06T05:28:50,176 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:50532 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741894_1078] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data4]'}, localName='127.0.0.1:38435', datanodeUuid='e97cdf8a-79a9-4085-ba36-a94e2f6ff519', xmitsInProgress=0}:Exception transferring block BP-2129024424-172.17.0.3-1733462887764:blk_1073741894_1078 to mirror 127.0.0.1:40573
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:50,177 WARN  [Thread-1024 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1078
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40573
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,177 WARN  [Thread-1024 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741894_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38435,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK], DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]) is bad.
2024-12-06T05:28:50,177 WARN  [Thread-1024 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741894_1078
2024-12-06T05:28:50,177 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:50532 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741894_1078] {}] datanode.BlockReceiver(316): Block 1073741894 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T05:28:50,177 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:50532 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741894_1078] {}] datanode.DataXceiver(331): 127.0.0.1:38435:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50532 dst: /127.0.0.1:38435
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:50,178 WARN  [Thread-1024 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]
2024-12-06T05:28:50,187 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741895_1079 (size=8190)
2024-12-06T05:28:50,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38435 is added to blk_1073741895_1079 (size=8190)
2024-12-06T05:28:50,188 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/eb98167d238b4108810871cb8d0e9d25
2024-12-06T05:28:50,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/eb98167d238b4108810871cb8d0e9d25 as hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/eb98167d238b4108810871cb8d0e9d25
2024-12-06T05:28:50,207 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/eb98167d238b4108810871cb8d0e9d25, entries=3, sequenceid=66, filesize=8.0 K
2024-12-06T05:28:50,208 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for a9e4791af9aa7cd7b9961aee9a47e3a5 in 45ms, sequenceid=66, compaction requested=false
2024-12-06T05:28:50,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9e4791af9aa7cd7b9961aee9a47e3a5:

2024-12-06T05:28:50,209 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K
2024-12-06T05:28:50,209 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:28:50,209 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/9dfd627ed399444a9a05bb0af7c086a4 because midkey is the same as first or last row
2024-12-06T05:28:50,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39957 {}] regionserver.HRegion(8855): Flush requested on a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:50,392 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9e4791af9aa7cd7b9961aee9a47e3a5 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-12-06T05:28:50,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/4fa414c234cd4ba2bf2a5d6c94305cc6 is 1080, key is row0015/info:/1733462930164/Put/seqid=0
2024-12-06T05:28:50,399 WARN  [Thread-1033 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741896_1080
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,399 WARN  [Thread-1033 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741896_1080 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK], DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]) is bad.
2024-12-06T05:28:50,399 WARN  [Thread-1033 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741896_1080
2024-12-06T05:28:50,399 WARN  [Thread-1033 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]
2024-12-06T05:28:50,401 WARN  [Thread-1033 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741897_1081
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40573
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,401 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:50552 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741897_1081] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data4]'}, localName='127.0.0.1:38435', datanodeUuid='e97cdf8a-79a9-4085-ba36-a94e2f6ff519', xmitsInProgress=0}:Exception transferring block BP-2129024424-172.17.0.3-1733462887764:blk_1073741897_1081 to mirror 127.0.0.1:40573
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:50,401 WARN  [Thread-1033 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741897_1081 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38435,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK], DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]) is bad.
2024-12-06T05:28:50,401 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:50552 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741897_1081] {}] datanode.BlockReceiver(316): Block 1073741897 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T05:28:50,401 WARN  [Thread-1033 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741897_1081
2024-12-06T05:28:50,401 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:50552 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741897_1081] {}] datanode.DataXceiver(331): 127.0.0.1:38435:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50552 dst: /127.0.0.1:38435
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:50,402 WARN  [Thread-1033 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]
2024-12-06T05:28:50,408 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,409 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-12-06T05:28:50,410 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741898_1082 (size=14660)
2024-12-06T05:28:50,410 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38435 is added to blk_1073741898_1082 (size=14660)
2024-12-06T05:28:50,446 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,558 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.1733462914379 to hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/oldWALs/ec46afd49254%2C39957%2C1733462890475.1733462914379
2024-12-06T05:28:50,592 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-06T05:28:50,592 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T05:28:50,593 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:28:50,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:50,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:50,593 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-06T05:28:50,593 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-06T05:28:50,593 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2055614761, stopped=false
2024-12-06T05:28:50,593 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ec46afd49254,34167,1733462890110
2024-12-06T05:28:50,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34771-0x101a91c92140002, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T05:28:50,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T05:28:50,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T05:28:50,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34771-0x101a91c92140002, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:50,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:50,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:50,664 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T05:28:50,664 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T05:28:50,664 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:28:50,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:50,664 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34771-0x101a91c92140002, quorum=127.0.0.1:55669, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:28:50,664 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:28:50,664 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ec46afd49254,39957,1733462890475' *****
2024-12-06T05:28:50,664 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-06T05:28:50,664 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ec46afd49254,34771,1733462892328' *****
2024-12-06T05:28:50,664 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-06T05:28:50,665 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-06T05:28:50,665 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-06T05:28:50,665 INFO  [RS:1;ec46afd49254:34771 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-06T05:28:50,665 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-06T05:28:50,665 INFO  [RS:1;ec46afd49254:34771 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-06T05:28:50,665 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.HRegionServer(959): stopping server ec46afd49254,34771,1733462892328
2024-12-06T05:28:50,665 INFO  [RS:1;ec46afd49254:34771 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T05:28:50,665 INFO  [RS:1;ec46afd49254:34771 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ec46afd49254:34771.
2024-12-06T05:28:50,665 DEBUG [RS:1;ec46afd49254:34771 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:28:50,665 DEBUG [RS:1;ec46afd49254:34771 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:50,665 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.HRegionServer(976): stopping server ec46afd49254,34771,1733462892328; all regions closed.
2024-12-06T05:28:50,666 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:28:50,666 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:50,666 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:50,666 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:50,666 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:50,666 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:50,667 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,667 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,667 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
2024-12-06T05:28:50,667 WARN  [IPC Server handler 2 on default port 41215 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575 has not been closed. Lease recovery is in progress. RecoveryId = 1083 for block blk_1073741837_1013
2024-12-06T05:28:50,668 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575 after 1ms
2024-12-06T05:28:50,811 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/4fa414c234cd4ba2bf2a5d6c94305cc6
2024-12-06T05:28:50,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/.tmp/info/4fa414c234cd4ba2bf2a5d6c94305cc6 as hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/4fa414c234cd4ba2bf2a5d6c94305cc6
2024-12-06T05:28:50,825 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/4fa414c234cd4ba2bf2a5d6c94305cc6, entries=9, sequenceid=79, filesize=14.3 K
2024-12-06T05:28:50,826 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10758, heapSize ~11.48 KB/11760, currentSize=0 B/0 for a9e4791af9aa7cd7b9961aee9a47e3a5 in 435ms, sequenceid=79, compaction requested=true
2024-12-06T05:28:50,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9e4791af9aa7cd7b9961aee9a47e3a5:

2024-12-06T05:28:50,826 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.0 K, sizeToCheck=16.0 K
2024-12-06T05:28:50,826 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:28:50,826 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/9dfd627ed399444a9a05bb0af7c086a4 because midkey is the same as first or last row
2024-12-06T05:28:50,826 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-06T05:28:50,826 INFO  [RS:0;ec46afd49254:39957 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-06T05:28:50,826 INFO  [RS:0;ec46afd49254:39957 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-06T05:28:50,826 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(3091): Received CLOSE for a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:50,827 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(959): stopping server ec46afd49254,39957,1733462890475
2024-12-06T05:28:50,827 INFO  [RS:0;ec46afd49254:39957 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T05:28:50,827 INFO  [RS:0;ec46afd49254:39957 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ec46afd49254:39957.
2024-12-06T05:28:50,827 DEBUG [RS:0;ec46afd49254:39957 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:28:50,827 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a9e4791af9aa7cd7b9961aee9a47e3a5, disabling compactions & flushes
2024-12-06T05:28:50,827 DEBUG [RS:0;ec46afd49254:39957 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:50,827 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.
2024-12-06T05:28:50,827 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-06T05:28:50,827 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.
2024-12-06T05:28:50,827 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-06T05:28:50,827 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-06T05:28:50,827 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5. after waiting 0 ms
2024-12-06T05:28:50,827 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.
2024-12-06T05:28:50,827 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-06T05:28:50,827 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-06T05:28:50,827 DEBUG [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(1325): Online Regions={a9e4791af9aa7cd7b9961aee9a47e3a5=TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5., 1588230740=hbase:meta,,1.1588230740}
2024-12-06T05:28:50,827 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T05:28:50,827 DEBUG [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, a9e4791af9aa7cd7b9961aee9a47e3a5
2024-12-06T05:28:50,827 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T05:28:50,828 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T05:28:50,827 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/f223a106daa44d0cba61ff99d5cf97d1, hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/705153fdc66c4999b49772d17d1beb95, hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/c6cfb60cfdd149278bbd0f1ce46d9c97, hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/b78e83e1f0d841fcb2ebdc994da2ab5f, hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/01d4cb4831f44595ad6146b725073a95, hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/bfe047c8a1d74ab0910c902e2dfeee49] to archive
2024-12-06T05:28:50,828 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T05:28:50,828 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T05:28:50,828 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-12-06T05:28:50,828 ERROR [FSHLog-0-hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086-prefix:ec46afd49254,39957,1733462890475.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,828 WARN  [FSHLog-0-hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086-prefix:ec46afd49254,39957,1733462890475.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,828 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ec46afd49254%2C39957%2C1733462890475.meta:.meta(num 1733462892145) roll requested
2024-12-06T05:28:50,828 INFO  [regionserver/ec46afd49254:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C39957%2C1733462890475.meta.1733462930828.meta
2024-12-06T05:28:50,829 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-06T05:28:50,831 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/f223a106daa44d0cba61ff99d5cf97d1 to hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/f223a106daa44d0cba61ff99d5cf97d1
2024-12-06T05:28:50,832 WARN  [Thread-1040 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741899_1084
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42285
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,832 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:44776 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741899_1084] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data8]'}, localName='127.0.0.1:41703', datanodeUuid='e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8', xmitsInProgress=0}:Exception transferring block BP-2129024424-172.17.0.3-1733462887764:blk_1073741899_1084 to mirror 127.0.0.1:42285
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:50,832 WARN  [Thread-1040 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741899_1084 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK], DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]) is bad.
2024-12-06T05:28:50,832 WARN  [Thread-1040 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741899_1084
2024-12-06T05:28:50,832 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:44776 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741899_1084] {}] datanode.BlockReceiver(316): Block 1073741899 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-06T05:28:50,832 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:44776 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741899_1084] {}] datanode.DataXceiver(331): 127.0.0.1:41703:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:44776 dst: /127.0.0.1:41703
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:50,833 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/705153fdc66c4999b49772d17d1beb95 to hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/705153fdc66c4999b49772d17d1beb95
2024-12-06T05:28:50,833 WARN  [Thread-1040 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]
2024-12-06T05:28:50,834 WARN  [Thread-1040 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741900_1085
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,834 WARN  [Thread-1040 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741900_1085 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK], DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]) is bad.
2024-12-06T05:28:50,834 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/c6cfb60cfdd149278bbd0f1ce46d9c97 to hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/c6cfb60cfdd149278bbd0f1ce46d9c97
2024-12-06T05:28:50,835 WARN  [Thread-1040 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741900_1085
2024-12-06T05:28:50,835 WARN  [Thread-1040 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]
2024-12-06T05:28:50,836 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/b78e83e1f0d841fcb2ebdc994da2ab5f to hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/b78e83e1f0d841fcb2ebdc994da2ab5f
2024-12-06T05:28:50,838 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/01d4cb4831f44595ad6146b725073a95 to hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/01d4cb4831f44595ad6146b725073a95
2024-12-06T05:28:50,840 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/bfe047c8a1d74ab0910c902e2dfeee49 to hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/info/bfe047c8a1d74ab0910c902e2dfeee49
2024-12-06T05:28:50,841 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ec46afd49254:34167 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-06T05:28:50,841 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [f223a106daa44d0cba61ff99d5cf97d1=10347, 705153fdc66c4999b49772d17d1beb95=12506, c6cfb60cfdd149278bbd0f1ce46d9c97=17994, b78e83e1f0d841fcb2ebdc994da2ab5f=6027, 01d4cb4831f44595ad6146b725073a95=6027, bfe047c8a1d74ab0910c902e2dfeee49=6027]
2024-12-06T05:28:50,845 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:50,845 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:50,845 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:50,845 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:50,845 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:50,845 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462930828.meta
2024-12-06T05:28:50,846 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,846 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,846 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
2024-12-06T05:28:50,847 WARN  [IPC Server handler 4 on default port 41215 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta has not been closed. Lease recovery is in progress. RecoveryId = 1087 for block blk_1073741834_1010
2024-12-06T05:28:50,847 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta after 1ms
2024-12-06T05:28:50,849 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39793:39793),(127.0.0.1/127.0.0.1:36195:36195)]
2024-12-06T05:28:50,849 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta is not closed yet, will try archiving it next time
2024-12-06T05:28:50,850 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a9e4791af9aa7cd7b9961aee9a47e3a5/recovered.edits/82.seqid, newMaxSeqId=82, maxSeqId=1
2024-12-06T05:28:50,850 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.
2024-12-06T05:28:50,851 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a9e4791af9aa7cd7b9961aee9a47e3a5:
Waiting for close lock at 1733462930827Running coprocessor pre-close hooks at 1733462930827Disabling compacts and flushes for region at 1733462930827Disabling writes for close at 1733462930827Writing region close event to WAL at 1733462930846 (+19 ms)Running coprocessor post-close hooks at 1733462930850 (+4 ms)Closed at 1733462930850
2024-12-06T05:28:50,851 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5.
2024-12-06T05:28:50,865 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740/.tmp/info/8cd4d015e4264a749f572c4f3f9d23a0 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1733462892469.a9e4791af9aa7cd7b9961aee9a47e3a5./info:regioninfo/1733462892842/Put/seqid=0
2024-12-06T05:28:50,868 WARN  [Thread-1048 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741902_1088
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,868 WARN  [Thread-1048 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741902_1088 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK], DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]) is bad.
2024-12-06T05:28:50,868 WARN  [Thread-1048 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741902_1088
2024-12-06T05:28:50,869 WARN  [Thread-1048 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41475,DS-7c9b59c6-aacb-41e1-821c-9291e799d6bd,DISK]
2024-12-06T05:28:50,873 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741903_1089 (size=7089)
2024-12-06T05:28:50,873 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38435 is added to blk_1073741903_1089 (size=7089)
2024-12-06T05:28:50,874 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740/.tmp/info/8cd4d015e4264a749f572c4f3f9d23a0
2024-12-06T05:28:50,896 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740/.tmp/ns/f4326dac922a4921a4e6b5e2b320a3e5 is 43, key is default/ns:d/1733462892226/Put/seqid=0
2024-12-06T05:28:50,899 WARN  [Thread-1054 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741904_1090
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42285
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,899 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:50576 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741904_1090] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data4]'}, localName='127.0.0.1:38435', datanodeUuid='e97cdf8a-79a9-4085-ba36-a94e2f6ff519', xmitsInProgress=0}:Exception transferring block BP-2129024424-172.17.0.3-1733462887764:blk_1073741904_1090 to mirror 127.0.0.1:42285
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:50,900 WARN  [Thread-1054 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741904_1090 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38435,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK], DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]) is bad.
2024-12-06T05:28:50,900 WARN  [Thread-1054 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741904_1090
2024-12-06T05:28:50,900 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:50576 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741904_1090] {}] datanode.BlockReceiver(316): Block 1073741904 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T05:28:50,900 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:50576 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741904_1090] {}] datanode.DataXceiver(331): 127.0.0.1:38435:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50576 dst: /127.0.0.1:38435
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:50,900 WARN  [Thread-1054 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]
2024-12-06T05:28:50,901 WARN  [Thread-1054 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741905_1091
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:50,902 WARN  [Thread-1054 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741905_1091 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK], DatanodeInfoWithStorage[127.0.0.1:41703,DS-04112c23-8d9c-40a2-af3f-0f38711c8d43,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]) is bad.
2024-12-06T05:28:50,902 WARN  [Thread-1054 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741905_1091
2024-12-06T05:28:50,902 WARN  [Thread-1054 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40573,DS-d59a34ad-75ad-4a8c-97f0-d752de8dccaa,DISK]
2024-12-06T05:28:50,907 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38435 is added to blk_1073741906_1092 (size=5153)
2024-12-06T05:28:50,907 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741906_1092 (size=5153)
2024-12-06T05:28:51,028 DEBUG [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-06T05:28:51,221 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4770dbe5[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41703, datanodeUuid=e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8, infoPort=39793, infoSecurePort=0, ipcPort=44537, storageInfo=lv=-57;cid=testClusterID;nsid=548480314;c=1733462887764):Failed to transfer BP-2129024424-172.17.0.3-1733462887764:blk_1073741874_1057 to 127.0.0.1:42285 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:51,228 DEBUG [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-06T05:28:51,308 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740/.tmp/ns/f4326dac922a4921a4e6b5e2b320a3e5
2024-12-06T05:28:51,338 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740/.tmp/table/42a890a4cce143bfb2fa3a6e89cf5e0a is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1733462892852/Put/seqid=0
2024-12-06T05:28:51,341 WARN  [Thread-1061 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741907_1093
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42285
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:28:51,341 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:50586 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741907_1093] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data4]'}, localName='127.0.0.1:38435', datanodeUuid='e97cdf8a-79a9-4085-ba36-a94e2f6ff519', xmitsInProgress=0}:Exception transferring block BP-2129024424-172.17.0.3-1733462887764:blk_1073741907_1093 to mirror 127.0.0.1:42285
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:51,341 WARN  [Thread-1061 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2129024424-172.17.0.3-1733462887764:blk_1073741907_1093 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38435,DS-ba1d5c22-4b0f-4f9e-9ba4-b141a3e97c96,DISK], DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]) is bad.
2024-12-06T05:28:51,341 WARN  [Thread-1061 {}] hdfs.DataStreamer(1850): Abandoning BP-2129024424-172.17.0.3-1733462887764:blk_1073741907_1093
2024-12-06T05:28:51,342 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:50586 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741907_1093] {}] datanode.BlockReceiver(316): Block 1073741907 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T05:28:51,342 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965057118_22 at /127.0.0.1:50586 [Receiving block BP-2129024424-172.17.0.3-1733462887764:blk_1073741907_1093] {}] datanode.DataXceiver(331): 127.0.0.1:38435:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50586 dst: /127.0.0.1:38435
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:51,342 WARN  [Thread-1061 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42285,DS-6c207a91-b278-477d-bd0b-5fb128ae6b31,DISK]
2024-12-06T05:28:51,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741908_1094 (size=5424)
2024-12-06T05:28:51,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38435 is added to blk_1073741908_1094 (size=5424)
2024-12-06T05:28:51,359 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740/.tmp/table/42a890a4cce143bfb2fa3a6e89cf5e0a
2024-12-06T05:28:51,368 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740/.tmp/info/8cd4d015e4264a749f572c4f3f9d23a0 as hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740/info/8cd4d015e4264a749f572c4f3f9d23a0
2024-12-06T05:28:51,375 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740/info/8cd4d015e4264a749f572c4f3f9d23a0, entries=10, sequenceid=11, filesize=6.9 K
2024-12-06T05:28:51,376 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740/.tmp/ns/f4326dac922a4921a4e6b5e2b320a3e5 as hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740/ns/f4326dac922a4921a4e6b5e2b320a3e5
2024-12-06T05:28:51,383 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740/ns/f4326dac922a4921a4e6b5e2b320a3e5, entries=2, sequenceid=11, filesize=5.0 K
2024-12-06T05:28:51,384 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740/.tmp/table/42a890a4cce143bfb2fa3a6e89cf5e0a as hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740/table/42a890a4cce143bfb2fa3a6e89cf5e0a
2024-12-06T05:28:51,393 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740/table/42a890a4cce143bfb2fa3a6e89cf5e0a, entries=2, sequenceid=11, filesize=5.3 K
2024-12-06T05:28:51,395 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 567ms, sequenceid=11, compaction requested=false
2024-12-06T05:28:51,402 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-06T05:28:51,403 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T05:28:51,403 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T05:28:51,404 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733462930827Running coprocessor pre-close hooks at 1733462930827Disabling compacts and flushes for region at 1733462930827Disabling writes for close at 1733462930828 (+1 ms)Obtaining lock to block concurrent updates at 1733462930828Preparing flush snapshotting stores in 1588230740 at 1733462930828Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1733462930828Flushing stores of hbase:meta,,1.1588230740 at 1733462930850 (+22 ms)Flushing 1588230740/info: creating writer at 1733462930850Flushing 1588230740/info: appending metadata at 1733462930865 (+15 ms)Flushing 1588230740/info: closing flushed file at 1733462930865Flushing 1588230740/ns: creating writer at 1733462930880 (+15 ms)Flushing 1588230740/ns: appending metadata at 1733462930895 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733462930895Flushing 1588230740/table: creating writer at 1733462931319 (+424 ms)Flushing 1588230740/table: appending metadata at 1733462931337 (+18 ms)Flushing 1588230740/table: closing flushed file at 1733462931337Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5303d62e: reopening flushed file at 1733462931367 (+30 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a1c097c: reopening flushed file at 1733462931375 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74432a85: reopening flushed file at 1733462931383 (+8 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 567ms, sequenceid=11, compaction requested=false at 1733462931395 (+12 ms)Writing region close event to WAL at 1733462931398 (+3 ms)Running coprocessor post-close hooks at 1733462931403 (+5 ms)Closed at 1733462931403
2024-12-06T05:28:51,404 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-06T05:28:51,428 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(976): stopping server ec46afd49254,39957,1733462890475; all regions closed.
2024-12-06T05:28:51,429 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:51,429 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:51,429 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:51,429 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:51,429 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:51,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38435 is added to blk_1073741901_1086 (size=825)
2024-12-06T05:28:51,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741901_1086 (size=825)
2024-12-06T05:28:51,440 INFO  [regionserver/ec46afd49254:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-06T05:28:51,440 INFO  [regionserver/ec46afd49254:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-06T05:28:51,650 INFO  [regionserver/ec46afd49254:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-06T05:28:51,651 INFO  [regionserver/ec46afd49254:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-06T05:28:51,653 INFO  [regionserver/ec46afd49254:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T05:28:52,409 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741833_1019 (size=455)
2024-12-06T05:28:52,442 INFO  [regionserver/ec46afd49254:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T05:28:52,883 INFO  [master/ec46afd49254:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-06T05:28:52,884 INFO  [master/ec46afd49254:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-06T05:28:54,375 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741832_1008 (size=32)
2024-12-06T05:28:54,375 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741836_1012 (size=76)
2024-12-06T05:28:54,669 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575 after 4002ms
2024-12-06T05:28:54,848 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta after 4002ms
2024-12-06T05:28:55,375 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741828_1004 (size=1189)
2024-12-06T05:28:55,377 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741826_1002 (size=42)
2024-12-06T05:28:55,667 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-06T05:28:55,669 DEBUG [RS:1;ec46afd49254:34771 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/oldWALs
2024-12-06T05:28:55,669 INFO  [RS:1;ec46afd49254:34771 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ec46afd49254%2C34771%2C1733462892328:(num 1733462892575)
2024-12-06T05:28:55,669 DEBUG [RS:1;ec46afd49254:34771 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:55,669 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T05:28:55,669 INFO  [RS:1;ec46afd49254:34771 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T05:28:55,669 INFO  [RS:1;ec46afd49254:34771 {}] hbase.ChoreService(370): Chore service for: regionserver/ec46afd49254:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-06T05:28:55,669 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-06T05:28:55,669 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T05:28:55,670 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-06T05:28:55,670 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-06T05:28:55,670 INFO  [RS:1;ec46afd49254:34771 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T05:28:55,670 INFO  [RS:1;ec46afd49254:34771 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34771
2024-12-06T05:28:55,673 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:28:55,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34771-0x101a91c92140002, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec46afd49254,34771,1733462892328
2024-12-06T05:28:55,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T05:28:55,716 INFO  [RS:1;ec46afd49254:34771 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T05:28:55,726 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec46afd49254,34771,1733462892328]
2024-12-06T05:28:55,737 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ec46afd49254,34771,1733462892328 already deleted, retry=false
2024-12-06T05:28:55,737 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ec46afd49254,34771,1733462892328 expired; onlineServers=1
2024-12-06T05:28:55,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34771-0x101a91c92140002, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:28:55,826 INFO  [RS:1;ec46afd49254:34771 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T05:28:55,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34771-0x101a91c92140002, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:28:55,826 INFO  [RS:1;ec46afd49254:34771 {}] regionserver.HRegionServer(1031): Exiting; stopping=ec46afd49254,34771,1733462892328; zookeeper connection closed.
2024-12-06T05:28:55,827 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@64ba2492 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@64ba2492
2024-12-06T05:28:55,852 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:55,867 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:55,867 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:55,868 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:55,868 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:55,868 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:55,879 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:55,880 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:56,382 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T05:28:56,397 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:56,397 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:56,397 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:56,398 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:56,398 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:56,398 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:56,402 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:56,405 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:28:56,430 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-06T05:28:56,433 DEBUG [RS:0;ec46afd49254:39957 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/oldWALs
2024-12-06T05:28:56,434 INFO  [RS:0;ec46afd49254:39957 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ec46afd49254%2C39957%2C1733462890475.meta:.meta(num 1733462930828)
2024-12-06T05:28:56,434 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:56,434 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:56,434 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:56,434 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:56,434 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:56,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38435 is added to blk_1073741891_1075 (size=15850)
2024-12-06T05:28:56,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741891_1075 (size=15850)
2024-12-06T05:28:56,439 DEBUG [RS:0;ec46afd49254:39957 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/oldWALs
2024-12-06T05:28:56,440 INFO  [RS:0;ec46afd49254:39957 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ec46afd49254%2C39957%2C1733462890475:(num 1733462930143)
2024-12-06T05:28:56,440 DEBUG [RS:0;ec46afd49254:39957 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:28:56,440 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T05:28:56,440 INFO  [RS:0;ec46afd49254:39957 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T05:28:56,440 INFO  [RS:0;ec46afd49254:39957 {}] hbase.ChoreService(370): Chore service for: regionserver/ec46afd49254:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-06T05:28:56,440 INFO  [RS:0;ec46afd49254:39957 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T05:28:56,440 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T05:28:56,440 INFO  [RS:0;ec46afd49254:39957 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39957
2024-12-06T05:28:56,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec46afd49254,39957,1733462890475
2024-12-06T05:28:56,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T05:28:56,484 INFO  [RS:0;ec46afd49254:39957 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T05:28:56,485 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec46afd49254,39957,1733462890475]
2024-12-06T05:28:56,505 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ec46afd49254,39957,1733462890475 already deleted, retry=false
2024-12-06T05:28:56,505 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ec46afd49254,39957,1733462890475 expired; onlineServers=0
2024-12-06T05:28:56,505 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-06T05:28:56,505 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ec46afd49254,34167,1733462890110' *****
2024-12-06T05:28:56,505 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-06T05:28:56,505 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T05:28:56,505 INFO  [M:0;ec46afd49254:34167 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T05:28:56,505 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-06T05:28:56,505 INFO  [M:0;ec46afd49254:34167 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T05:28:56,506 DEBUG [M:0;ec46afd49254:34167 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-06T05:28:56,506 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-06T05:28:56,506 DEBUG [M:0;ec46afd49254:34167 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-06T05:28:56,506 DEBUG [master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462891503 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462891503,5,FailOnTimeoutGroup]
2024-12-06T05:28:56,506 DEBUG [master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462891502 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462891502,5,FailOnTimeoutGroup]
2024-12-06T05:28:56,506 INFO  [M:0;ec46afd49254:34167 {}] hbase.ChoreService(370): Chore service for: master/ec46afd49254:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-06T05:28:56,506 INFO  [M:0;ec46afd49254:34167 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T05:28:56,506 DEBUG [M:0;ec46afd49254:34167 {}] master.HMaster(1795): Stopping service threads
2024-12-06T05:28:56,506 INFO  [M:0;ec46afd49254:34167 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-06T05:28:56,506 INFO  [M:0;ec46afd49254:34167 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T05:28:56,507 INFO  [M:0;ec46afd49254:34167 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-06T05:28:56,507 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-06T05:28:56,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-06T05:28:56,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:28:56,516 DEBUG [M:0;ec46afd49254:34167 {}] zookeeper.ZKUtil(347): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-06T05:28:56,516 WARN  [M:0;ec46afd49254:34167 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-06T05:28:56,517 INFO  [M:0;ec46afd49254:34167 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/.lastflushedseqids
2024-12-06T05:28:56,522 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38435 is added to blk_1073741909_1095 (size=130)
2024-12-06T05:28:56,522 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741909_1095 (size=130)
2024-12-06T05:28:56,523 INFO  [M:0;ec46afd49254:34167 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-06T05:28:56,523 INFO  [M:0;ec46afd49254:34167 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-06T05:28:56,523 DEBUG [M:0;ec46afd49254:34167 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T05:28:56,523 INFO  [M:0;ec46afd49254:34167 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:56,523 DEBUG [M:0;ec46afd49254:34167 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:56,523 DEBUG [M:0;ec46afd49254:34167 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T05:28:56,523 DEBUG [M:0;ec46afd49254:34167 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:56,523 INFO  [M:0;ec46afd49254:34167 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.25 KB heapSize=29.49 KB
2024-12-06T05:28:56,539 DEBUG [M:0;ec46afd49254:34167 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7f9e5649837e4843b6e927bbdbb0ec6c is 82, key is hbase:meta,,1/info:regioninfo/1733462892175/Put/seqid=0
2024-12-06T05:28:56,544 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741910_1096 (size=5672)
2024-12-06T05:28:56,544 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38435 is added to blk_1073741910_1096 (size=5672)
2024-12-06T05:28:56,544 INFO  [M:0;ec46afd49254:34167 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7f9e5649837e4843b6e927bbdbb0ec6c
2024-12-06T05:28:56,566 DEBUG [M:0;ec46afd49254:34167 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/12b4820dfa89474c8b2dc462a3ebacb6 is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733462892858/Put/seqid=0
2024-12-06T05:28:56,571 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38435 is added to blk_1073741911_1097 (size=6255)
2024-12-06T05:28:56,571 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741911_1097 (size=6255)
2024-12-06T05:28:56,572 INFO  [M:0;ec46afd49254:34167 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/12b4820dfa89474c8b2dc462a3ebacb6
2024-12-06T05:28:56,577 INFO  [M:0;ec46afd49254:34167 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 12b4820dfa89474c8b2dc462a3ebacb6
2024-12-06T05:28:56,594 DEBUG [M:0;ec46afd49254:34167 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7df10af830c54cbaac426025f7afa6ea is 69, key is ec46afd49254,34771,1733462892328/rs:state/1733462892409/Put/seqid=0
2024-12-06T05:28:56,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:28:56,595 INFO  [RS:0;ec46afd49254:39957 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T05:28:56,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x101a91c92140001, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:28:56,595 INFO  [RS:0;ec46afd49254:39957 {}] regionserver.HRegionServer(1031): Exiting; stopping=ec46afd49254,39957,1733462890475; zookeeper connection closed.
2024-12-06T05:28:56,595 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6486b96c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6486b96c
2024-12-06T05:28:56,595 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-12-06T05:28:56,599 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38435 is added to blk_1073741912_1098 (size=5224)
2024-12-06T05:28:56,599 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741912_1098 (size=5224)
2024-12-06T05:28:56,599 INFO  [M:0;ec46afd49254:34167 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7df10af830c54cbaac426025f7afa6ea
2024-12-06T05:28:56,622 DEBUG [M:0;ec46afd49254:34167 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5b988eaf8ab54605bafdb56b62fda0b5 is 52, key is load_balancer_on/state:d/1733462892313/Put/seqid=0
2024-12-06T05:28:56,627 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741913_1099 (size=5056)
2024-12-06T05:28:56,627 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38435 is added to blk_1073741913_1099 (size=5056)
2024-12-06T05:28:56,627 INFO  [M:0;ec46afd49254:34167 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5b988eaf8ab54605bafdb56b62fda0b5
2024-12-06T05:28:56,634 DEBUG [M:0;ec46afd49254:34167 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7f9e5649837e4843b6e927bbdbb0ec6c as hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7f9e5649837e4843b6e927bbdbb0ec6c
2024-12-06T05:28:56,646 INFO  [M:0;ec46afd49254:34167 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7f9e5649837e4843b6e927bbdbb0ec6c, entries=8, sequenceid=60, filesize=5.5 K
2024-12-06T05:28:56,648 DEBUG [M:0;ec46afd49254:34167 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/12b4820dfa89474c8b2dc462a3ebacb6 as hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/12b4820dfa89474c8b2dc462a3ebacb6
2024-12-06T05:28:56,653 INFO  [M:0;ec46afd49254:34167 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 12b4820dfa89474c8b2dc462a3ebacb6
2024-12-06T05:28:56,654 INFO  [M:0;ec46afd49254:34167 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/12b4820dfa89474c8b2dc462a3ebacb6, entries=6, sequenceid=60, filesize=6.1 K
2024-12-06T05:28:56,655 DEBUG [M:0;ec46afd49254:34167 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7df10af830c54cbaac426025f7afa6ea as hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7df10af830c54cbaac426025f7afa6ea
2024-12-06T05:28:56,663 INFO  [M:0;ec46afd49254:34167 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7df10af830c54cbaac426025f7afa6ea, entries=2, sequenceid=60, filesize=5.1 K
2024-12-06T05:28:56,665 DEBUG [M:0;ec46afd49254:34167 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5b988eaf8ab54605bafdb56b62fda0b5 as hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5b988eaf8ab54605bafdb56b62fda0b5
2024-12-06T05:28:56,673 INFO  [M:0;ec46afd49254:34167 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5b988eaf8ab54605bafdb56b62fda0b5, entries=1, sequenceid=60, filesize=4.9 K
2024-12-06T05:28:56,674 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:28:56,675 INFO  [M:0;ec46afd49254:34167 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 151ms, sequenceid=60, compaction requested=false
2024-12-06T05:28:56,677 INFO  [M:0;ec46afd49254:34167 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:28:56,677 DEBUG [M:0;ec46afd49254:34167 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733462936523Disabling compacts and flushes for region at 1733462936523Disabling writes for close at 1733462936523Obtaining lock to block concurrent updates at 1733462936523Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733462936523Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23805, getHeapSize=30136, getOffHeapSize=0, getCellsCount=71 at 1733462936524 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733462936524Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733462936524Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733462936539 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733462936539Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733462936551 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733462936565 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733462936565Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733462936577 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733462936593 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733462936593Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733462936606 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733462936621 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733462936621Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24c8a4ac: reopening flushed file at 1733462936633 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ad6b5b1: reopening flushed file at 1733462936647 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47d001d2: reopening flushed file at 1733462936654 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d979469: reopening flushed file at 1733462936663 (+9 ms)Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 151ms, sequenceid=60, compaction requested=false at 1733462936675 (+12 ms)Writing region close event to WAL at 1733462936677 (+2 ms)Closed at 1733462936677
2024-12-06T05:28:56,678 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:56,678 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:56,679 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:56,679 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:56,679 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:28:56,681 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741888_1071 (size=1045)
2024-12-06T05:28:56,681 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38435 is added to blk_1073741888_1071 (size=1045)
2024-12-06T05:28:56,849 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:28:57,397 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@60c4f636 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-2129024424-172.17.0.3-1733462887764:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:41475,null,null])
java.net.ConnectException: Call From ec46afd49254/172.17.0.3 to localhost:46841 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-06T05:28:57,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741835_1011 (size=393)
2024-12-06T05:28:57,399 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741831_1007 (size=1321)
2024-12-06T05:28:57,537 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/WALs/ec46afd49254,34167,1733462890110/ec46afd49254%2C34167%2C1733462890110.1733462890673 to hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/oldWALs/ec46afd49254%2C34167%2C1733462890110.1733462890673
2024-12-06T05:28:57,542 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/MasterData/oldWALs/ec46afd49254%2C34167%2C1733462890110.1733462890673 to hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/oldWALs/ec46afd49254%2C34167%2C1733462890110.1733462890673$masterlocalwal$
2024-12-06T05:28:57,542 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T05:28:57,542 INFO  [M:0;ec46afd49254:34167 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-06T05:28:57,542 INFO  [M:0;ec46afd49254:34167 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34167
2024-12-06T05:28:57,542 INFO  [M:0;ec46afd49254:34167 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T05:28:57,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:28:57,658 INFO  [M:0;ec46afd49254:34167 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T05:28:57,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34167-0x101a91c92140000, quorum=127.0.0.1:55669, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:28:57,663 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@15b530e2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:57,664 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7f6709ee{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:28:57,664 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:28:57,664 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@557202e0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:28:57,664 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54f27916{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.log.dir/,STOPPED}
2024-12-06T05:28:57,667 WARN  [BP-2129024424-172.17.0.3-1733462887764 heartbeating to localhost/127.0.0.1:41215 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:28:57,667 WARN  [BP-2129024424-172.17.0.3-1733462887764 heartbeating to localhost/127.0.0.1:41215 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2129024424-172.17.0.3-1733462887764 (Datanode Uuid e97cdf8a-79a9-4085-ba36-a94e2f6ff519) service to localhost/127.0.0.1:41215
2024-12-06T05:28:57,667 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@398dae23 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-2129024424-172.17.0.3-1733462887764:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:41475,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:46841 , LocalHost:localPort ec46afd49254/172.17.0.3:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-06T05:28:57,668 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@398dae23 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-2129024424-172.17.0.3-1733462887764:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:38435,null,null])
java.io.IOException: No block pool offer service for bpid=BP-2129024424-172.17.0.3-1733462887764
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:57,668 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@398dae23 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-2129024424-172.17.0.3-1733462887764:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:41475,null,null], DatanodeInfoWithStorage[127.0.0.1:38435,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-2129024424-172.17.0.3-1733462887764:blk_1073741837_1013, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:41475,null,null], DatanodeInfoWithStorage[127.0.0.1:38435,null,null]]
2024-12-06T05:28:57,668 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@398dae23 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-2129024424-172.17.0.3-1733462887764:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:41475,null,null])
java.io.IOException: No block pool offer service for bpid=BP-2129024424-172.17.0.3-1733462887764
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:57,669 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:28:57,669 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data3/current/BP-2129024424-172.17.0.3-1733462887764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:57,669 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@398dae23 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-2129024424-172.17.0.3-1733462887764:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:38435,null,null])
java.io.IOException: No block pool offer service for bpid=BP-2129024424-172.17.0.3-1733462887764
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:28:57,669 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:28:57,669 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@398dae23 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-2129024424-172.17.0.3-1733462887764:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:41475,null,null], DatanodeInfoWithStorage[127.0.0.1:38435,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-2129024424-172.17.0.3-1733462887764:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:41475,null,null], DatanodeInfoWithStorage[127.0.0.1:38435,null,null]]
2024-12-06T05:28:57,669 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data4/current/BP-2129024424-172.17.0.3-1733462887764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:57,675 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:28:57,700 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:28:57,703 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6079f97a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:57,703 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1196c8fc{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:28:57,703 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:28:57,703 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7cde9b58{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:28:57,703 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@514acf4b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.log.dir/,STOPPED}
2024-12-06T05:28:57,706 WARN  [BP-2129024424-172.17.0.3-1733462887764 heartbeating to localhost/127.0.0.1:41215 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:28:57,706 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:28:57,706 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:28:57,706 WARN  [BP-2129024424-172.17.0.3-1733462887764 heartbeating to localhost/127.0.0.1:41215 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2129024424-172.17.0.3-1733462887764 (Datanode Uuid e4ab7dac-f7ff-4c56-8c48-9dc8c67fcaf8) service to localhost/127.0.0.1:41215
2024-12-06T05:28:57,706 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data7/current/BP-2129024424-172.17.0.3-1733462887764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:57,706 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/cluster_6d4fa5ca-9134-03e1-0894-9da30f3da145/data/data8/current/BP-2129024424-172.17.0.3-1733462887764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:28:57,707 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:28:57,713 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2606b08f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T05:28:57,713 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@c053989{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:28:57,713 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:28:57,713 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3150e6db{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:28:57,714 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1aa07d80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.log.dir/,STOPPED}
2024-12-06T05:28:57,722 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-06T05:28:57,750 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-06T05:28:57,758 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=155 (was 79)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41215 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:41215
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41215
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:35237
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41215
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:41215
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:35237
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:41215
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007fc700bf4000.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41215 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41215
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41215
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007fc700bf4000.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41215 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41215 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=448 (was 404) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=233 (was 127) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6961 (was 7640)
2024-12-06T05:28:57,764 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=155, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=233, ProcessCount=11, AvailableMemoryMB=6962
2024-12-06T05:28:57,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-06T05:28:57,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.log.dir so I do NOT create it in target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc
2024-12-06T05:28:57,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/303b4513-1a35-b118-9c45-ad6e4b6d0642/hadoop.tmp.dir so I do NOT create it in target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc
2024-12-06T05:28:57,765 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75, deleteOnExit=true
2024-12-06T05:28:57,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-06T05:28:57,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/test.cache.data in system properties and HBase conf
2024-12-06T05:28:57,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.tmp.dir in system properties and HBase conf
2024-12-06T05:28:57,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.log.dir in system properties and HBase conf
2024-12-06T05:28:57,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-06T05:28:57,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-06T05:28:57,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-06T05:28:57,766 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-06T05:28:57,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-06T05:28:57,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-06T05:28:57,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-06T05:28:57,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T05:28:57,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-06T05:28:57,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-06T05:28:57,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T05:28:57,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T05:28:57,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-06T05:28:57,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/nfs.dump.dir in system properties and HBase conf
2024-12-06T05:28:57,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/java.io.tmpdir in system properties and HBase conf
2024-12-06T05:28:57,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T05:28:57,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-06T05:28:57,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-06T05:28:57,777 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T05:28:57,850 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:28:58,136 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:28:58,142 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:28:58,144 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:28:58,144 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:28:58,144 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T05:28:58,145 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:28:58,145 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@61f9169f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:28:58,145 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11255fea{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:28:58,250 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@586e8021{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/java.io.tmpdir/jetty-localhost-44221-hadoop-hdfs-3_4_1-tests_jar-_-any-18088054389782643592/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T05:28:58,250 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7ec5b9af{HTTP/1.1, (http/1.1)}{localhost:44221}
2024-12-06T05:28:58,250 INFO  [Time-limited test {}] server.Server(415): Started @157861ms
2024-12-06T05:28:58,262 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T05:28:58,501 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:28:58,505 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:28:58,506 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:28:58,506 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:28:58,506 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T05:28:58,508 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@123783d2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:28:58,508 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47ffea33{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:28:58,597 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2cfd21d9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/java.io.tmpdir/jetty-localhost-34691-hadoop-hdfs-3_4_1-tests_jar-_-any-7196032946484128460/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:58,598 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c3df9c{HTTP/1.1, (http/1.1)}{localhost:34691}
2024-12-06T05:28:58,598 INFO  [Time-limited test {}] server.Server(415): Started @158209ms
2024-12-06T05:28:58,599 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:28:58,622 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:28:58,625 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:28:58,626 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:28:58,626 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:28:58,626 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T05:28:58,627 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54dbaae8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:28:58,627 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@37b5496d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:28:58,676 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:28:58,717 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5960bf29{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/java.io.tmpdir/jetty-localhost-46669-hadoop-hdfs-3_4_1-tests_jar-_-any-5473287313293214316/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:28:58,717 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@752fee05{HTTP/1.1, (http/1.1)}{localhost:46669}
2024-12-06T05:28:58,717 INFO  [Time-limited test {}] server.Server(415): Started @158328ms
2024-12-06T05:28:58,718 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:28:58,851 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:28:59,676 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:28:59,852 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:28:59,995 WARN  [Thread-1202 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/data/data1/current/BP-374932401-172.17.0.3-1733462937788/current, will proceed with Du for space computation calculation, 
2024-12-06T05:28:59,996 WARN  [Thread-1203 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/data/data2/current/BP-374932401-172.17.0.3-1733462937788/current, will proceed with Du for space computation calculation, 
2024-12-06T05:29:00,011 WARN  [Thread-1166 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:29:00,014 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbc9582ca2d0275f7 with lease ID 0xcdafecd5498cbddd: Processing first storage report for DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58 from datanode DatanodeRegistration(127.0.0.1:33335, datanodeUuid=1d187868-3ce7-467a-aaf4-09d22fe90c6d, infoPort=34035, infoSecurePort=0, ipcPort=41063, storageInfo=lv=-57;cid=testClusterID;nsid=1369661529;c=1733462937788)
2024-12-06T05:29:00,014 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbc9582ca2d0275f7 with lease ID 0xcdafecd5498cbddd: from storage DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58 node DatanodeRegistration(127.0.0.1:33335, datanodeUuid=1d187868-3ce7-467a-aaf4-09d22fe90c6d, infoPort=34035, infoSecurePort=0, ipcPort=41063, storageInfo=lv=-57;cid=testClusterID;nsid=1369661529;c=1733462937788), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:29:00,014 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbc9582ca2d0275f7 with lease ID 0xcdafecd5498cbddd: Processing first storage report for DS-bc6490d7-bcfd-4b63-ba23-0fbff276b251 from datanode DatanodeRegistration(127.0.0.1:33335, datanodeUuid=1d187868-3ce7-467a-aaf4-09d22fe90c6d, infoPort=34035, infoSecurePort=0, ipcPort=41063, storageInfo=lv=-57;cid=testClusterID;nsid=1369661529;c=1733462937788)
2024-12-06T05:29:00,014 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbc9582ca2d0275f7 with lease ID 0xcdafecd5498cbddd: from storage DS-bc6490d7-bcfd-4b63-ba23-0fbff276b251 node DatanodeRegistration(127.0.0.1:33335, datanodeUuid=1d187868-3ce7-467a-aaf4-09d22fe90c6d, infoPort=34035, infoSecurePort=0, ipcPort=41063, storageInfo=lv=-57;cid=testClusterID;nsid=1369661529;c=1733462937788), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:29:00,129 WARN  [Thread-1213 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/data/data3/current/BP-374932401-172.17.0.3-1733462937788/current, will proceed with Du for space computation calculation, 
2024-12-06T05:29:00,129 WARN  [Thread-1214 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/data/data4/current/BP-374932401-172.17.0.3-1733462937788/current, will proceed with Du for space computation calculation, 
2024-12-06T05:29:00,151 WARN  [Thread-1189 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:29:00,153 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e1e69c2230020c2 with lease ID 0xcdafecd5498cbdde: Processing first storage report for DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af from datanode DatanodeRegistration(127.0.0.1:37657, datanodeUuid=2976047c-a66b-4db7-8cdd-428d3778c51a, infoPort=33005, infoSecurePort=0, ipcPort=34141, storageInfo=lv=-57;cid=testClusterID;nsid=1369661529;c=1733462937788)
2024-12-06T05:29:00,153 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e1e69c2230020c2 with lease ID 0xcdafecd5498cbdde: from storage DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af node DatanodeRegistration(127.0.0.1:37657, datanodeUuid=2976047c-a66b-4db7-8cdd-428d3778c51a, infoPort=33005, infoSecurePort=0, ipcPort=34141, storageInfo=lv=-57;cid=testClusterID;nsid=1369661529;c=1733462937788), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:29:00,153 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e1e69c2230020c2 with lease ID 0xcdafecd5498cbdde: Processing first storage report for DS-86440bd0-79b5-46b6-b1cb-24cb0d1ace18 from datanode DatanodeRegistration(127.0.0.1:37657, datanodeUuid=2976047c-a66b-4db7-8cdd-428d3778c51a, infoPort=33005, infoSecurePort=0, ipcPort=34141, storageInfo=lv=-57;cid=testClusterID;nsid=1369661529;c=1733462937788)
2024-12-06T05:29:00,153 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e1e69c2230020c2 with lease ID 0xcdafecd5498cbdde: from storage DS-86440bd0-79b5-46b6-b1cb-24cb0d1ace18 node DatanodeRegistration(127.0.0.1:37657, datanodeUuid=2976047c-a66b-4db7-8cdd-428d3778c51a, infoPort=33005, infoSecurePort=0, ipcPort=34141, storageInfo=lv=-57;cid=testClusterID;nsid=1369661529;c=1733462937788), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:29:00,256 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc
2024-12-06T05:29:00,259 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/zookeeper_0, clientPort=63652, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-06T05:29:00,260 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63652
2024-12-06T05:29:00,261 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:29:00,262 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:29:00,275 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741825_1001 (size=7)
2024-12-06T05:29:00,275 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33335 is added to blk_1073741825_1001 (size=7)
2024-12-06T05:29:00,277 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373 with version=8
2024-12-06T05:29:00,277 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/hbase-staging
2024-12-06T05:29:00,279 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/ec46afd49254:0 server-side Connection retries=45
2024-12-06T05:29:00,279 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:29:00,279 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T05:29:00,279 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T05:29:00,279 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:29:00,279 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T05:29:00,280 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-06T05:29:00,280 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T05:29:00,280 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43165
2024-12-06T05:29:00,282 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43165 connecting to ZooKeeper ensemble=127.0.0.1:63652
2024-12-06T05:29:00,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:431650x0, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T05:29:00,338 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43165-0x101a91d560e0000 connected
2024-12-06T05:29:00,420 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:29:00,424 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:29:00,429 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:29:00,429 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373, hbase.cluster.distributed=false
2024-12-06T05:29:00,431 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T05:29:00,431 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43165
2024-12-06T05:29:00,431 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43165
2024-12-06T05:29:00,432 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43165
2024-12-06T05:29:00,432 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43165
2024-12-06T05:29:00,432 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43165
2024-12-06T05:29:00,445 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/ec46afd49254:0 server-side Connection retries=45
2024-12-06T05:29:00,446 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:29:00,446 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T05:29:00,446 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T05:29:00,446 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:29:00,446 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T05:29:00,446 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-06T05:29:00,446 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T05:29:00,447 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38219
2024-12-06T05:29:00,447 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38219 connecting to ZooKeeper ensemble=127.0.0.1:63652
2024-12-06T05:29:00,448 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:29:00,449 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:29:00,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382190x0, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T05:29:00,461 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38219-0x101a91d560e0001 connected
2024-12-06T05:29:00,461 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:29:00,461 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-06T05:29:00,462 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-06T05:29:00,462 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-06T05:29:00,463 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T05:29:00,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38219
2024-12-06T05:29:00,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38219
2024-12-06T05:29:00,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38219
2024-12-06T05:29:00,465 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38219
2024-12-06T05:29:00,465 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38219
2024-12-06T05:29:00,478 DEBUG [M:0;ec46afd49254:43165 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ec46afd49254:43165
2024-12-06T05:29:00,478 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ec46afd49254,43165,1733462940279
2024-12-06T05:29:00,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:29:00,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:29:00,491 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ec46afd49254,43165,1733462940279
2024-12-06T05:29:00,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-06T05:29:00,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:00,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:00,504 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-06T05:29:00,504 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ec46afd49254,43165,1733462940279 from backup master directory
2024-12-06T05:29:00,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ec46afd49254,43165,1733462940279
2024-12-06T05:29:00,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:29:00,514 WARN  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T05:29:00,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:29:00,514 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ec46afd49254,43165,1733462940279
2024-12-06T05:29:00,523 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/hbase.id] with ID: 2dd00f65-f3db-483e-8b8d-45be21f77495
2024-12-06T05:29:00,523 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/.tmp/hbase.id
2024-12-06T05:29:00,529 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741826_1002 (size=42)
2024-12-06T05:29:00,529 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33335 is added to blk_1073741826_1002 (size=42)
2024-12-06T05:29:00,530 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/.tmp/hbase.id]:[hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/hbase.id]
2024-12-06T05:29:00,547 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:29:00,547 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-06T05:29:00,549 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-12-06T05:29:00,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:00,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:00,563 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741827_1003 (size=196)
2024-12-06T05:29:00,563 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33335 is added to blk_1073741827_1003 (size=196)
2024-12-06T05:29:00,564 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T05:29:00,564 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-06T05:29:00,565 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:29:00,571 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33335 is added to blk_1073741828_1004 (size=1189)
2024-12-06T05:29:00,572 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741828_1004 (size=1189)
2024-12-06T05:29:00,572 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store
2024-12-06T05:29:00,581 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741829_1005 (size=34)
2024-12-06T05:29:00,581 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33335 is added to blk_1073741829_1005 (size=34)
2024-12-06T05:29:00,581 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:29:00,581 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T05:29:00,582 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:29:00,582 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:29:00,582 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T05:29:00,582 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:29:00,582 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:29:00,582 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733462940581Disabling compacts and flushes for region at 1733462940581Disabling writes for close at 1733462940582 (+1 ms)Writing region close event to WAL at 1733462940582Closed at 1733462940582
2024-12-06T05:29:00,582 WARN  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/.initializing
2024-12-06T05:29:00,582 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/WALs/ec46afd49254,43165,1733462940279
2024-12-06T05:29:00,585 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C43165%2C1733462940279, suffix=, logDir=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/WALs/ec46afd49254,43165,1733462940279, archiveDir=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/oldWALs, maxLogs=10
2024-12-06T05:29:00,585 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C43165%2C1733462940279.1733462940585
2024-12-06T05:29:00,590 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/WALs/ec46afd49254,43165,1733462940279/ec46afd49254%2C43165%2C1733462940279.1733462940585
2024-12-06T05:29:00,590 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34035:34035),(127.0.0.1/127.0.0.1:33005:33005)]
2024-12-06T05:29:00,591 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:29:00,591 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:29:00,591 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:29:00,591 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:29:00,592 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:29:00,593 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-06T05:29:00,593 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:00,594 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:00,594 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:29:00,595 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-06T05:29:00,595 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:00,595 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:29:00,595 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:29:00,596 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-06T05:29:00,596 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:00,597 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:29:00,597 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:29:00,598 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-06T05:29:00,598 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:00,599 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:29:00,599 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:29:00,600 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:29:00,600 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:29:00,601 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:29:00,601 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:29:00,602 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-06T05:29:00,602 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:29:00,604 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:29:00,605 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=866882, jitterRate=0.10229851305484772}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-06T05:29:00,605 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733462940591Initializing all the Stores at 1733462940592 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462940592Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462940592Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462940592Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462940592Cleaning up temporary data from old regions at 1733462940601 (+9 ms)Region opened successfully at 1733462940605 (+4 ms)
2024-12-06T05:29:00,606 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-06T05:29:00,609 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57a2b1d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0
2024-12-06T05:29:00,609 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-06T05:29:00,610 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-06T05:29:00,610 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-06T05:29:00,610 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-06T05:29:00,610 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-06T05:29:00,610 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-06T05:29:00,610 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-06T05:29:00,613 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-06T05:29:00,614 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-06T05:29:00,618 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-06T05:29:00,619 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-06T05:29:00,619 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-06T05:29:00,629 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-06T05:29:00,629 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-06T05:29:00,630 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-06T05:29:00,640 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-06T05:29:00,642 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-06T05:29:00,650 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-06T05:29:00,653 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-06T05:29:00,660 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-06T05:29:00,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T05:29:00,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T05:29:00,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:00,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:00,672 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ec46afd49254,43165,1733462940279, sessionid=0x101a91d560e0000, setting cluster-up flag (Was=false)
2024-12-06T05:29:00,677 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:00,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:00,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:00,724 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-06T05:29:00,727 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec46afd49254,43165,1733462940279
2024-12-06T05:29:00,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:00,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:00,777 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-06T05:29:00,781 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec46afd49254,43165,1733462940279
2024-12-06T05:29:00,785 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-06T05:29:00,788 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-06T05:29:00,788 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-06T05:29:00,788 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-06T05:29:00,789 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ec46afd49254,43165,1733462940279
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-06T05:29:00,790 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:29:00,791 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:29:00,791 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:29:00,791 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:29:00,791 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ec46afd49254:0, corePoolSize=10, maxPoolSize=10
2024-12-06T05:29:00,791 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:00,791 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ec46afd49254:0, corePoolSize=2, maxPoolSize=2
2024-12-06T05:29:00,791 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:00,792 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733462970792
2024-12-06T05:29:00,792 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-06T05:29:00,792 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-06T05:29:00,792 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-06T05:29:00,792 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-06T05:29:00,792 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-06T05:29:00,792 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-06T05:29:00,793 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:00,793 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-06T05:29:00,793 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-06T05:29:00,793 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:29:00,793 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-06T05:29:00,793 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-06T05:29:00,794 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-06T05:29:00,794 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-06T05:29:00,794 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462940794,5,FailOnTimeoutGroup]
2024-12-06T05:29:00,794 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462940794,5,FailOnTimeoutGroup]
2024-12-06T05:29:00,794 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:00,794 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-06T05:29:00,794 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:00,794 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:00,795 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:00,795 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-06T05:29:00,802 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33335 is added to blk_1073741831_1007 (size=1321)
2024-12-06T05:29:00,802 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741831_1007 (size=1321)
2024-12-06T05:29:00,803 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-06T05:29:00,803 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373
2024-12-06T05:29:00,809 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741832_1008 (size=32)
2024-12-06T05:29:00,809 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33335 is added to blk_1073741832_1008 (size=32)
2024-12-06T05:29:00,810 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:29:00,811 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T05:29:00,812 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T05:29:00,812 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:00,812 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:00,812 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T05:29:00,813 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T05:29:00,813 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:00,814 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:00,814 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T05:29:00,815 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T05:29:00,815 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:00,815 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:00,815 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T05:29:00,816 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T05:29:00,816 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:00,817 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:00,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T05:29:00,818 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740
2024-12-06T05:29:00,818 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740
2024-12-06T05:29:00,819 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T05:29:00,819 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T05:29:00,820 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T05:29:00,821 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T05:29:00,823 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:29:00,823 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=845715, jitterRate=0.07538236677646637}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T05:29:00,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733462940810Initializing all the Stores at 1733462940810Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462940810Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462940811 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462940811Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462940811Cleaning up temporary data from old regions at 1733462940819 (+8 ms)Region opened successfully at 1733462940824 (+5 ms)
2024-12-06T05:29:00,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T05:29:00,824 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T05:29:00,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T05:29:00,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T05:29:00,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T05:29:00,824 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T05:29:00,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733462940824Disabling compacts and flushes for region at 1733462940824Disabling writes for close at 1733462940824Writing region close event to WAL at 1733462940824Closed at 1733462940824
2024-12-06T05:29:00,826 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:29:00,826 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-06T05:29:00,826 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-06T05:29:00,827 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T05:29:00,828 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-06T05:29:00,853 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:00,867 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.HRegionServer(746): ClusterId : 2dd00f65-f3db-483e-8b8d-45be21f77495
2024-12-06T05:29:00,867 DEBUG [RS:0;ec46afd49254:38219 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-06T05:29:00,872 DEBUG [RS:0;ec46afd49254:38219 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-06T05:29:00,872 DEBUG [RS:0;ec46afd49254:38219 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-06T05:29:00,883 DEBUG [RS:0;ec46afd49254:38219 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-06T05:29:00,883 DEBUG [RS:0;ec46afd49254:38219 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a240886, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0
2024-12-06T05:29:00,900 DEBUG [RS:0;ec46afd49254:38219 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ec46afd49254:38219
2024-12-06T05:29:00,901 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-06T05:29:00,901 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-06T05:29:00,901 DEBUG [RS:0;ec46afd49254:38219 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-06T05:29:00,902 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec46afd49254,43165,1733462940279 with port=38219, startcode=1733462940445
2024-12-06T05:29:00,902 DEBUG [RS:0;ec46afd49254:38219 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-06T05:29:00,904 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60157, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-12-06T05:29:00,904 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43165 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ec46afd49254,38219,1733462940445
2024-12-06T05:29:00,904 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43165 {}] master.ServerManager(517): Registering regionserver=ec46afd49254,38219,1733462940445
2024-12-06T05:29:00,906 DEBUG [RS:0;ec46afd49254:38219 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373
2024-12-06T05:29:00,906 DEBUG [RS:0;ec46afd49254:38219 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45395
2024-12-06T05:29:00,906 DEBUG [RS:0;ec46afd49254:38219 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-06T05:29:00,978 WARN  [ec46afd49254:43165 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-06T05:29:00,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T05:29:00,992 DEBUG [RS:0;ec46afd49254:38219 {}] zookeeper.ZKUtil(111): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec46afd49254,38219,1733462940445
2024-12-06T05:29:00,993 WARN  [RS:0;ec46afd49254:38219 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T05:29:00,993 INFO  [RS:0;ec46afd49254:38219 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:29:00,993 DEBUG [RS:0;ec46afd49254:38219 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445
2024-12-06T05:29:00,994 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec46afd49254,38219,1733462940445]
2024-12-06T05:29:01,002 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-06T05:29:01,005 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-06T05:29:01,005 INFO  [RS:0;ec46afd49254:38219 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T05:29:01,005 INFO  [RS:0;ec46afd49254:38219 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:01,005 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-06T05:29:01,006 INFO  [RS:0;ec46afd49254:38219 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-06T05:29:01,006 INFO  [RS:0;ec46afd49254:38219 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:01,006 DEBUG [RS:0;ec46afd49254:38219 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:01,006 DEBUG [RS:0;ec46afd49254:38219 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:01,007 DEBUG [RS:0;ec46afd49254:38219 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:01,007 DEBUG [RS:0;ec46afd49254:38219 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:01,007 DEBUG [RS:0;ec46afd49254:38219 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:01,007 DEBUG [RS:0;ec46afd49254:38219 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0, corePoolSize=2, maxPoolSize=2
2024-12-06T05:29:01,007 DEBUG [RS:0;ec46afd49254:38219 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:01,007 DEBUG [RS:0;ec46afd49254:38219 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:01,007 DEBUG [RS:0;ec46afd49254:38219 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:01,007 DEBUG [RS:0;ec46afd49254:38219 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:01,007 DEBUG [RS:0;ec46afd49254:38219 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:01,007 DEBUG [RS:0;ec46afd49254:38219 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:01,007 DEBUG [RS:0;ec46afd49254:38219 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3
2024-12-06T05:29:01,007 DEBUG [RS:0;ec46afd49254:38219 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3
2024-12-06T05:29:01,008 INFO  [RS:0;ec46afd49254:38219 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:01,008 INFO  [RS:0;ec46afd49254:38219 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:01,008 INFO  [RS:0;ec46afd49254:38219 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:01,008 INFO  [RS:0;ec46afd49254:38219 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:01,008 INFO  [RS:0;ec46afd49254:38219 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:01,008 INFO  [RS:0;ec46afd49254:38219 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,38219,1733462940445-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T05:29:01,024 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-06T05:29:01,024 INFO  [RS:0;ec46afd49254:38219 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,38219,1733462940445-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:01,024 INFO  [RS:0;ec46afd49254:38219 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:01,024 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.Replication(171): ec46afd49254,38219,1733462940445 started
2024-12-06T05:29:01,036 INFO  [RS:0;ec46afd49254:38219 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:01,036 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.HRegionServer(1482): Serving as ec46afd49254,38219,1733462940445, RpcServer on ec46afd49254/172.17.0.3:38219, sessionid=0x101a91d560e0001
2024-12-06T05:29:01,036 DEBUG [RS:0;ec46afd49254:38219 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-06T05:29:01,036 DEBUG [RS:0;ec46afd49254:38219 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec46afd49254,38219,1733462940445
2024-12-06T05:29:01,036 DEBUG [RS:0;ec46afd49254:38219 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,38219,1733462940445'
2024-12-06T05:29:01,036 DEBUG [RS:0;ec46afd49254:38219 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-06T05:29:01,037 DEBUG [RS:0;ec46afd49254:38219 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-06T05:29:01,037 DEBUG [RS:0;ec46afd49254:38219 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-06T05:29:01,037 DEBUG [RS:0;ec46afd49254:38219 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-06T05:29:01,037 DEBUG [RS:0;ec46afd49254:38219 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec46afd49254,38219,1733462940445
2024-12-06T05:29:01,037 DEBUG [RS:0;ec46afd49254:38219 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,38219,1733462940445'
2024-12-06T05:29:01,037 DEBUG [RS:0;ec46afd49254:38219 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-06T05:29:01,038 DEBUG [RS:0;ec46afd49254:38219 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-06T05:29:01,038 DEBUG [RS:0;ec46afd49254:38219 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-06T05:29:01,038 INFO  [RS:0;ec46afd49254:38219 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-06T05:29:01,038 INFO  [RS:0;ec46afd49254:38219 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-06T05:29:01,143 INFO  [RS:0;ec46afd49254:38219 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C38219%2C1733462940445, suffix=, logDir=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445, archiveDir=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/oldWALs, maxLogs=32
2024-12-06T05:29:01,144 INFO  [RS:0;ec46afd49254:38219 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C38219%2C1733462940445.1733462941143
2024-12-06T05:29:01,151 INFO  [RS:0;ec46afd49254:38219 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462941143
2024-12-06T05:29:01,152 DEBUG [RS:0;ec46afd49254:38219 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34035:34035),(127.0.0.1/127.0.0.1:33005:33005)]
2024-12-06T05:29:01,228 DEBUG [ec46afd49254:43165 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-06T05:29:01,229 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ec46afd49254,38219,1733462940445
2024-12-06T05:29:01,232 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec46afd49254,38219,1733462940445, state=OPENING
2024-12-06T05:29:01,259 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-06T05:29:01,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:01,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:01,348 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T05:29:01,349 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:29:01,349 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ec46afd49254,38219,1733462940445}]
2024-12-06T05:29:01,349 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:29:01,503 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-06T05:29:01,508 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36653, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-06T05:29:01,514 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-06T05:29:01,515 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:29:01,517 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C38219%2C1733462940445.meta, suffix=.meta, logDir=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445, archiveDir=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/oldWALs, maxLogs=32
2024-12-06T05:29:01,517 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C38219%2C1733462940445.meta.1733462941517.meta
2024-12-06T05:29:01,523 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.meta.1733462941517.meta
2024-12-06T05:29:01,524 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33005:33005),(127.0.0.1/127.0.0.1:34035:34035)]
2024-12-06T05:29:01,525 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:29:01,525 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-06T05:29:01,525 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-06T05:29:01,525 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-06T05:29:01,525 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-06T05:29:01,526 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:29:01,526 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-06T05:29:01,526 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-06T05:29:01,527 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T05:29:01,528 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T05:29:01,528 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:01,529 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:01,529 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T05:29:01,530 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T05:29:01,530 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:01,530 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:01,531 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T05:29:01,531 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T05:29:01,532 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:01,532 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:01,532 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T05:29:01,533 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T05:29:01,533 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:01,534 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:01,534 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T05:29:01,535 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740
2024-12-06T05:29:01,536 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740
2024-12-06T05:29:01,537 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T05:29:01,537 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T05:29:01,538 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T05:29:01,539 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T05:29:01,540 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=753581, jitterRate=-0.04177311062812805}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T05:29:01,540 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-06T05:29:01,540 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733462941526Writing region info on filesystem at 1733462941526Initializing all the Stores at 1733462941527 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462941527Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462941527Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462941527Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462941527Cleaning up temporary data from old regions at 1733462941537 (+10 ms)Running coprocessor post-open hooks at 1733462941540 (+3 ms)Region opened successfully at 1733462941540
2024-12-06T05:29:01,541 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733462941502
2024-12-06T05:29:01,543 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-06T05:29:01,543 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-06T05:29:01,544 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ec46afd49254,38219,1733462940445
2024-12-06T05:29:01,545 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec46afd49254,38219,1733462940445, state=OPEN
2024-12-06T05:29:01,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T05:29:01,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T05:29:01,600 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ec46afd49254,38219,1733462940445
2024-12-06T05:29:01,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:29:01,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:29:01,607 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-06T05:29:01,607 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec46afd49254,38219,1733462940445 in 251 msec
2024-12-06T05:29:01,611 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-06T05:29:01,611 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 781 msec
2024-12-06T05:29:01,612 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:29:01,612 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-06T05:29:01,614 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T05:29:01,614 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec46afd49254,38219,1733462940445, seqNum=-1]
2024-12-06T05:29:01,614 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T05:29:01,615 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49287, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T05:29:01,624 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 835 msec
2024-12-06T05:29:01,624 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733462941624, completionTime=-1
2024-12-06T05:29:01,624 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-06T05:29:01,624 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-06T05:29:01,627 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-06T05:29:01,627 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733463001627
2024-12-06T05:29:01,627 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733463061627
2024-12-06T05:29:01,627 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-06T05:29:01,627 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43165,1733462940279-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:01,627 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43165,1733462940279-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:01,627 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43165,1733462940279-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:01,628 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ec46afd49254:43165, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:01,628 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:01,628 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:01,630 DEBUG [master/ec46afd49254:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T05:29:01,633 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.119sec
2024-12-06T05:29:01,633 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-06T05:29:01,633 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-06T05:29:01,633 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-06T05:29:01,633 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-06T05:29:01,633 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-06T05:29:01,633 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43165,1733462940279-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T05:29:01,633 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43165,1733462940279-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-06T05:29:01,637 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-06T05:29:01,637 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-06T05:29:01,637 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43165,1733462940279-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:01,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70770496, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:29:01,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ec46afd49254,43165,-1 for getting cluster id
2024-12-06T05:29:01,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-06T05:29:01,672 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2dd00f65-f3db-483e-8b8d-45be21f77495'
2024-12-06T05:29:01,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-06T05:29:01,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2dd00f65-f3db-483e-8b8d-45be21f77495"

2024-12-06T05:29:01,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117426b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:29:01,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ec46afd49254,43165,-1]
2024-12-06T05:29:01,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-06T05:29:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:29:01,676 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-06T05:29:01,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4254f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:29:01,678 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:01,678 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T05:29:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec46afd49254,38219,1733462940445, seqNum=-1]
2024-12-06T05:29:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T05:29:01,682 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T05:29:01,684 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ec46afd49254,43165,1733462940279
2024-12-06T05:29:01,685 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:29:01,688 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-06T05:29:01,688 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-12-06T05:29:01,688 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-12-06T05:29:01,689 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-06T05:29:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is ec46afd49254,43165,1733462940279
2024-12-06T05:29:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3812bc96
2024-12-06T05:29:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-06T05:29:01,693 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-06T05:29:01,694 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43165 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-06T05:29:01,694 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43165 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-06T05:29:01,694 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43165 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T05:29:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43165 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-12-06T05:29:01,698 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-12-06T05:29:01,698 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:01,698 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43165 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-12-06T05:29:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T05:29:01,700 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-06T05:29:01,708 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33335 is added to blk_1073741835_1011 (size=395)
2024-12-06T05:29:01,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741835_1011 (size=395)
2024-12-06T05:29:01,711 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5b3a8be16cfff4418b5ccc392be46d27, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373
2024-12-06T05:29:01,718 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741836_1012 (size=78)
2024-12-06T05:29:01,719 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33335 is added to blk_1073741836_1012 (size=78)
2024-12-06T05:29:01,720 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:29:01,720 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 5b3a8be16cfff4418b5ccc392be46d27, disabling compactions & flushes
2024-12-06T05:29:01,720 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.
2024-12-06T05:29:01,720 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.
2024-12-06T05:29:01,720 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27. after waiting 0 ms
2024-12-06T05:29:01,720 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.
2024-12-06T05:29:01,720 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.
2024-12-06T05:29:01,720 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5b3a8be16cfff4418b5ccc392be46d27:
Waiting for close lock at 1733462941720Disabling compacts and flushes for region at 1733462941720Disabling writes for close at 1733462941720Writing region close event to WAL at 1733462941720Closed at 1733462941720
2024-12-06T05:29:01,722 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-12-06T05:29:01,722 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1733462941722"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733462941722"}]},"ts":"1733462941722"}
2024-12-06T05:29:01,725 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-06T05:29:01,727 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-06T05:29:01,727 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733462941727"}]},"ts":"1733462941727"}
2024-12-06T05:29:01,730 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-12-06T05:29:01,730 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=5b3a8be16cfff4418b5ccc392be46d27, ASSIGN}]
2024-12-06T05:29:01,732 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=5b3a8be16cfff4418b5ccc392be46d27, ASSIGN
2024-12-06T05:29:01,733 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=5b3a8be16cfff4418b5ccc392be46d27, ASSIGN; state=OFFLINE, location=ec46afd49254,38219,1733462940445; forceNewPlan=false, retain=false
2024-12-06T05:29:01,854 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:01,885 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5b3a8be16cfff4418b5ccc392be46d27, regionState=OPENING, regionLocation=ec46afd49254,38219,1733462940445
2024-12-06T05:29:01,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=5b3a8be16cfff4418b5ccc392be46d27, ASSIGN because future has completed
2024-12-06T05:29:01,892 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b3a8be16cfff4418b5ccc392be46d27, server=ec46afd49254,38219,1733462940445}]
2024-12-06T05:29:02,011 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T05:29:02,032 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:02,032 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:02,032 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:02,033 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:02,033 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:02,033 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:02,036 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:02,036 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:02,036 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:02,038 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:02,049 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.
2024-12-06T05:29:02,049 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5b3a8be16cfff4418b5ccc392be46d27, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:29:02,050 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 5b3a8be16cfff4418b5ccc392be46d27
2024-12-06T05:29:02,050 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:29:02,050 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5b3a8be16cfff4418b5ccc392be46d27
2024-12-06T05:29:02,050 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5b3a8be16cfff4418b5ccc392be46d27
2024-12-06T05:29:02,051 INFO  [StoreOpener-5b3a8be16cfff4418b5ccc392be46d27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 5b3a8be16cfff4418b5ccc392be46d27 
2024-12-06T05:29:02,052 INFO  [StoreOpener-5b3a8be16cfff4418b5ccc392be46d27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b3a8be16cfff4418b5ccc392be46d27 columnFamilyName info
2024-12-06T05:29:02,052 DEBUG [StoreOpener-5b3a8be16cfff4418b5ccc392be46d27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:02,053 INFO  [StoreOpener-5b3a8be16cfff4418b5ccc392be46d27-1 {}] regionserver.HStore(327): Store=5b3a8be16cfff4418b5ccc392be46d27/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:29:02,053 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5b3a8be16cfff4418b5ccc392be46d27
2024-12-06T05:29:02,054 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/default/TestLogRolling-testLogRollOnPipelineRestart/5b3a8be16cfff4418b5ccc392be46d27
2024-12-06T05:29:02,054 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/default/TestLogRolling-testLogRollOnPipelineRestart/5b3a8be16cfff4418b5ccc392be46d27
2024-12-06T05:29:02,054 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5b3a8be16cfff4418b5ccc392be46d27
2024-12-06T05:29:02,054 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5b3a8be16cfff4418b5ccc392be46d27
2024-12-06T05:29:02,056 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5b3a8be16cfff4418b5ccc392be46d27
2024-12-06T05:29:02,058 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/default/TestLogRolling-testLogRollOnPipelineRestart/5b3a8be16cfff4418b5ccc392be46d27/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:29:02,058 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5b3a8be16cfff4418b5ccc392be46d27; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=704363, jitterRate=-0.10435695946216583}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-06T05:29:02,058 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5b3a8be16cfff4418b5ccc392be46d27
2024-12-06T05:29:02,059 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5b3a8be16cfff4418b5ccc392be46d27:
Running coprocessor pre-open hook at 1733462942050Writing region info on filesystem at 1733462942050Initializing all the Stores at 1733462942051 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462942051Cleaning up temporary data from old regions at 1733462942054 (+3 ms)Running coprocessor post-open hooks at 1733462942058 (+4 ms)Region opened successfully at 1733462942059 (+1 ms)
2024-12-06T05:29:02,060 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27., pid=6, masterSystemTime=1733462942045
2024-12-06T05:29:02,062 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.
2024-12-06T05:29:02,062 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.
2024-12-06T05:29:02,063 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5b3a8be16cfff4418b5ccc392be46d27, regionState=OPEN, openSeqNum=2, regionLocation=ec46afd49254,38219,1733462940445
2024-12-06T05:29:02,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b3a8be16cfff4418b5ccc392be46d27, server=ec46afd49254,38219,1733462940445 because future has completed
2024-12-06T05:29:02,070 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-06T05:29:02,070 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5b3a8be16cfff4418b5ccc392be46d27, server=ec46afd49254,38219,1733462940445 in 175 msec
2024-12-06T05:29:02,073 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-06T05:29:02,073 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=5b3a8be16cfff4418b5ccc392be46d27, ASSIGN in 340 msec
2024-12-06T05:29:02,074 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-06T05:29:02,074 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733462942074"}]},"ts":"1733462942074"}
2024-12-06T05:29:02,076 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-12-06T05:29:02,077 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-12-06T05:29:02,080 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 383 msec
2024-12-06T05:29:02,679 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:02,856 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:03,680 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:03,857 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:04,681 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:04,858 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:05,682 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:05,859 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:06,505 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-06T05:29:06,505 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-06T05:29:06,506 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-06T05:29:06,506 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-12-06T05:29:06,507 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T05:29:06,507 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-06T05:29:06,683 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:06,860 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:07,030 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T05:29:07,052 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:07,053 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:07,053 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:07,053 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:07,053 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:07,054 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:07,056 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:07,056 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:07,057 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:07,058 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:07,062 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-06T05:29:07,062 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-12-06T05:29:07,684 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:07,861 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:08,686 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:08,863 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:09,687 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:09,864 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:10,688 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:10,865 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:11,690 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:11,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T05:29:11,728 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-12-06T05:29:11,729 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-12-06T05:29:11,736 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-12-06T05:29:11,736 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.
2024-12-06T05:29:11,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27., hostname=ec46afd49254,38219,1733462940445, seqNum=2]
2024-12-06T05:29:11,866 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:12,691 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:12,867 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:13,692 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:13,744 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462941143
2024-12-06T05:29:13,744 WARN  [ResponseProcessor for block BP-374932401-172.17.0.3-1733462937788:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-374932401-172.17.0.3-1733462937788:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:13,744 WARN  [ResponseProcessor for block BP-374932401-172.17.0.3-1733462937788:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-374932401-172.17.0.3-1733462937788:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-374932401-172.17.0.3-1733462937788:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:37657,DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:13,745 WARN  [DataStreamer for file /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462941143 block BP-374932401-172.17.0.3-1733462937788:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-374932401-172.17.0.3-1733462937788:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33335,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK], DatanodeInfoWithStorage[127.0.0.1:37657,DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37657,DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af,DISK]) is bad.
2024-12-06T05:29:13,745 WARN  [ResponseProcessor for block BP-374932401-172.17.0.3-1733462937788:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-374932401-172.17.0.3-1733462937788:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-374932401-172.17.0.3-1733462937788:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:37657,DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:13,745 WARN  [DataStreamer for file /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/WALs/ec46afd49254,43165,1733462940279/ec46afd49254%2C43165%2C1733462940279.1733462940585 block BP-374932401-172.17.0.3-1733462937788:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-374932401-172.17.0.3-1733462937788:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33335,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK], DatanodeInfoWithStorage[127.0.0.1:37657,DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37657,DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af,DISK]) is bad.
2024-12-06T05:29:13,745 WARN  [PacketResponder: BP-374932401-172.17.0.3-1733462937788:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:37657] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:13,745 WARN  [PacketResponder: BP-374932401-172.17.0.3-1733462937788:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:37657] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:13,746 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1757691826_22 at /127.0.0.1:45148 [Receiving block BP-374932401-172.17.0.3-1733462937788:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33335:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45148 dst: /127.0.0.1:33335
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:13,746 WARN  [DataStreamer for file /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.meta.1733462941517.meta block BP-374932401-172.17.0.3-1733462937788:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-374932401-172.17.0.3-1733462937788:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37657,DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af,DISK], DatanodeInfoWithStorage[127.0.0.1:33335,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37657,DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af,DISK]) is bad.
2024-12-06T05:29:13,746 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1757691826_22 at /127.0.0.1:44730 [Receiving block BP-374932401-172.17.0.3-1733462937788:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:37657:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:44730 dst: /127.0.0.1:37657
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:13,746 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1489413447_22 at /127.0.0.1:44712 [Receiving block BP-374932401-172.17.0.3-1733462937788:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:37657:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:44712 dst: /127.0.0.1:37657
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:13,746 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1489413447_22 at /127.0.0.1:45128 [Receiving block BP-374932401-172.17.0.3-1733462937788:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33335:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45128 dst: /127.0.0.1:33335
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:13,746 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1757691826_22 at /127.0.0.1:44732 [Receiving block BP-374932401-172.17.0.3-1733462937788:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:37657:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:44732 dst: /127.0.0.1:37657
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:13,746 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1757691826_22 at /127.0.0.1:45150 [Receiving block BP-374932401-172.17.0.3-1733462937788:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33335:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45150 dst: /127.0.0.1:33335
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:13,774 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5960bf29{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:29:13,774 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@752fee05{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:29:13,774 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:29:13,774 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@37b5496d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:29:13,775 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54dbaae8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.log.dir/,STOPPED}
2024-12-06T05:29:13,776 WARN  [BP-374932401-172.17.0.3-1733462937788 heartbeating to localhost/127.0.0.1:45395 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:29:13,776 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:29:13,776 WARN  [BP-374932401-172.17.0.3-1733462937788 heartbeating to localhost/127.0.0.1:45395 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-374932401-172.17.0.3-1733462937788 (Datanode Uuid 2976047c-a66b-4db7-8cdd-428d3778c51a) service to localhost/127.0.0.1:45395
2024-12-06T05:29:13,776 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:29:13,776 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/data/data3/current/BP-374932401-172.17.0.3-1733462937788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:29:13,777 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/data/data4/current/BP-374932401-172.17.0.3-1733462937788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:29:13,777 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:29:13,795 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:29:13,800 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:29:13,807 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:29:13,807 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:29:13,807 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T05:29:13,808 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5f1a012{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:29:13,808 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@117b424d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:29:13,868 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:13,923 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2647e756{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/java.io.tmpdir/jetty-localhost-46359-hadoop-hdfs-3_4_1-tests_jar-_-any-13921460065025835247/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:29:13,926 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@443f9e40{HTTP/1.1, (http/1.1)}{localhost:46359}
2024-12-06T05:29:13,927 INFO  [Time-limited test {}] server.Server(415): Started @173538ms
2024-12-06T05:29:13,928 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:29:13,979 WARN  [ResponseProcessor for block BP-374932401-172.17.0.3-1733462937788:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-374932401-172.17.0.3-1733462937788:blk_1073741830_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:13,979 WARN  [ResponseProcessor for block BP-374932401-172.17.0.3-1733462937788:blk_1073741834_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-374932401-172.17.0.3-1733462937788:blk_1073741834_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:13,979 WARN  [ResponseProcessor for block BP-374932401-172.17.0.3-1733462937788:blk_1073741833_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-374932401-172.17.0.3-1733462937788:blk_1073741833_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:13,980 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1489413447_22 at /127.0.0.1:53590 [Receiving block BP-374932401-172.17.0.3-1733462937788:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33335:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53590 dst: /127.0.0.1:33335
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:13,980 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1757691826_22 at /127.0.0.1:53610 [Receiving block BP-374932401-172.17.0.3-1733462937788:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33335:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53610 dst: /127.0.0.1:33335
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:13,980 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1757691826_22 at /127.0.0.1:53592 [Receiving block BP-374932401-172.17.0.3-1733462937788:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33335:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53592 dst: /127.0.0.1:33335
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:13,995 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2cfd21d9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:29:13,996 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c3df9c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:29:13,996 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:29:13,996 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47ffea33{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:29:13,996 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@123783d2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.log.dir/,STOPPED}
2024-12-06T05:29:13,998 WARN  [BP-374932401-172.17.0.3-1733462937788 heartbeating to localhost/127.0.0.1:45395 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:29:13,998 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:29:13,998 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:29:13,998 WARN  [BP-374932401-172.17.0.3-1733462937788 heartbeating to localhost/127.0.0.1:45395 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-374932401-172.17.0.3-1733462937788 (Datanode Uuid 1d187868-3ce7-467a-aaf4-09d22fe90c6d) service to localhost/127.0.0.1:45395
2024-12-06T05:29:13,999 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/data/data1/current/BP-374932401-172.17.0.3-1733462937788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:29:13,999 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/data/data2/current/BP-374932401-172.17.0.3-1733462937788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:29:13,999 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:29:14,019 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:29:14,026 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:29:14,028 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:29:14,029 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:29:14,029 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T05:29:14,032 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5d802677{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:29:14,032 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b559376{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:29:14,131 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@929fb8a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/java.io.tmpdir/jetty-localhost-38159-hadoop-hdfs-3_4_1-tests_jar-_-any-17150186044343009132/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:29:14,131 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@39f8899d{HTTP/1.1, (http/1.1)}{localhost:38159}
2024-12-06T05:29:14,131 INFO  [Time-limited test {}] server.Server(415): Started @173742ms
2024-12-06T05:29:14,133 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:29:14,543 WARN  [Thread-1338 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:29:14,547 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf6449f392bd81c5f with lease ID 0xcdafecd5498cbddf: from storage DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af node DatanodeRegistration(127.0.0.1:38877, datanodeUuid=2976047c-a66b-4db7-8cdd-428d3778c51a, infoPort=33623, infoSecurePort=0, ipcPort=44033, storageInfo=lv=-57;cid=testClusterID;nsid=1369661529;c=1733462937788), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-06T05:29:14,548 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf6449f392bd81c5f with lease ID 0xcdafecd5498cbddf: from storage DS-86440bd0-79b5-46b6-b1cb-24cb0d1ace18 node DatanodeRegistration(127.0.0.1:38877, datanodeUuid=2976047c-a66b-4db7-8cdd-428d3778c51a, infoPort=33623, infoSecurePort=0, ipcPort=44033, storageInfo=lv=-57;cid=testClusterID;nsid=1369661529;c=1733462937788), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:29:14,690 WARN  [Thread-1358 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:29:14,692 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x90b0af14e48158bc with lease ID 0xcdafecd5498cbde0: from storage DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58 node DatanodeRegistration(127.0.0.1:39257, datanodeUuid=1d187868-3ce7-467a-aaf4-09d22fe90c6d, infoPort=34653, infoSecurePort=0, ipcPort=35067, storageInfo=lv=-57;cid=testClusterID;nsid=1369661529;c=1733462937788), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:29:14,693 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x90b0af14e48158bc with lease ID 0xcdafecd5498cbde0: from storage DS-bc6490d7-bcfd-4b63-ba23-0fbff276b251 node DatanodeRegistration(127.0.0.1:39257, datanodeUuid=1d187868-3ce7-467a-aaf4-09d22fe90c6d, infoPort=34653, infoSecurePort=0, ipcPort=35067, storageInfo=lv=-57;cid=testClusterID;nsid=1369661529;c=1733462937788), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:29:14,693 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:14,869 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:15,160 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-12-06T05:29:15,163 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-12-06T05:29:15,164 ERROR [FSHLog-0-hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373-prefix:ec46afd49254,38219,1733462940445 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33335,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:15,164 WARN  [FSHLog-0-hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373-prefix:ec46afd49254,38219,1733462940445 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33335,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:15,164 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ec46afd49254%2C38219%2C1733462940445:(num 1733462941143) roll requested
2024-12-06T05:29:15,165 INFO  [regionserver/ec46afd49254:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C38219%2C1733462940445.1733462955164
2024-12-06T05:29:15,170 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462941143 newFile=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462955164
2024-12-06T05:29:15,170 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:15,170 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:15,171 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:15,171 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:15,171 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:15,171 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462941143 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462955164
2024-12-06T05:29:15,171 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33335,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:15,171 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33335,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:15,171 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462941143
2024-12-06T05:29:15,172 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:34653:34653)]
2024-12-06T05:29:15,172 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462941143 is not closed yet, will try archiving it next time
2024-12-06T05:29:15,172 WARN  [IPC Server handler 4 on default port 45395 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462941143 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1013
2024-12-06T05:29:15,172 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462941143 after 1ms
2024-12-06T05:29:15,693 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:15,869 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:16,546 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1013: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-06T05:29:16,694 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:16,870 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:17,175 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-12-06T05:29:17,695 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:17,871 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:18,696 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:18,872 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:19,173 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462941143 after 4002ms
2024-12-06T05:29:19,178 WARN  [ResponseProcessor for block BP-374932401-172.17.0.3-1733462937788:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-374932401-172.17.0.3-1733462937788:blk_1073741837_1016
java.io.IOException: Bad response ERROR for BP-374932401-172.17.0.3-1733462937788:blk_1073741837_1016 from datanode DatanodeInfoWithStorage[127.0.0.1:39257,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:19,179 WARN  [PacketResponder: BP-374932401-172.17.0.3-1733462937788:blk_1073741837_1016, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39257] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:19,179 WARN  [DataStreamer for file /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462955164 block BP-374932401-172.17.0.3-1733462937788:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-374932401-172.17.0.3-1733462937788:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38877,DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af,DISK], DatanodeInfoWithStorage[127.0.0.1:39257,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39257,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK]) is bad.
2024-12-06T05:29:19,179 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1757691826_22 at /127.0.0.1:35542 [Receiving block BP-374932401-172.17.0.3-1733462937788:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:38877:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35542 dst: /127.0.0.1:38877
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:19,180 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1757691826_22 at /127.0.0.1:44822 [Receiving block BP-374932401-172.17.0.3-1733462937788:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:39257:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:44822 dst: /127.0.0.1:39257
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:19,183 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@929fb8a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:29:19,184 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@39f8899d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:29:19,184 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:29:19,184 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b559376{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:29:19,184 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5d802677{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.log.dir/,STOPPED}
2024-12-06T05:29:19,186 WARN  [BP-374932401-172.17.0.3-1733462937788 heartbeating to localhost/127.0.0.1:45395 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:29:19,186 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:29:19,186 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:29:19,186 WARN  [BP-374932401-172.17.0.3-1733462937788 heartbeating to localhost/127.0.0.1:45395 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-374932401-172.17.0.3-1733462937788 (Datanode Uuid 1d187868-3ce7-467a-aaf4-09d22fe90c6d) service to localhost/127.0.0.1:45395
2024-12-06T05:29:19,187 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/data/data1/current/BP-374932401-172.17.0.3-1733462937788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:29:19,187 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/data/data2/current/BP-374932401-172.17.0.3-1733462937788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:29:19,187 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:29:19,201 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:29:19,205 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:29:19,206 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:29:19,207 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:29:19,207 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T05:29:19,208 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a79eeb5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:29:19,209 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d237fac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:29:19,318 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@48919a97{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/java.io.tmpdir/jetty-localhost-46143-hadoop-hdfs-3_4_1-tests_jar-_-any-9076205133527423630/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:29:19,319 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4778d192{HTTP/1.1, (http/1.1)}{localhost:46143}
2024-12-06T05:29:19,319 INFO  [Time-limited test {}] server.Server(415): Started @178930ms
2024-12-06T05:29:19,320 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:29:19,335 WARN  [ResponseProcessor for block BP-374932401-172.17.0.3-1733462937788:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-374932401-172.17.0.3-1733462937788:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:19,335 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1757691826_22 at /127.0.0.1:35560 [Receiving block BP-374932401-172.17.0.3-1733462937788:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:38877:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35560 dst: /127.0.0.1:38877
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:19,339 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2647e756{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:29:19,339 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@443f9e40{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:29:19,339 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:29:19,339 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@117b424d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:29:19,340 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5f1a012{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.log.dir/,STOPPED}
2024-12-06T05:29:19,340 WARN  [BP-374932401-172.17.0.3-1733462937788 heartbeating to localhost/127.0.0.1:45395 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:29:19,341 WARN  [BP-374932401-172.17.0.3-1733462937788 heartbeating to localhost/127.0.0.1:45395 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-374932401-172.17.0.3-1733462937788 (Datanode Uuid 2976047c-a66b-4db7-8cdd-428d3778c51a) service to localhost/127.0.0.1:45395
2024-12-06T05:29:19,341 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:29:19,341 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:29:19,341 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/data/data3/current/BP-374932401-172.17.0.3-1733462937788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:29:19,341 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/data/data4/current/BP-374932401-172.17.0.3-1733462937788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:29:19,341 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:29:19,349 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:29:19,352 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:29:19,353 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:29:19,353 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:29:19,353 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T05:29:19,353 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@160f7ba9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:29:19,354 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fc558ce{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:29:19,462 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@39e17fd3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/java.io.tmpdir/jetty-localhost-42447-hadoop-hdfs-3_4_1-tests_jar-_-any-315024924577153566/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:29:19,463 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5b16a6e2{HTTP/1.1, (http/1.1)}{localhost:42447}
2024-12-06T05:29:19,463 INFO  [Time-limited test {}] server.Server(415): Started @179074ms
2024-12-06T05:29:19,464 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:29:19,697 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:19,873 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:19,959 WARN  [Thread-1412 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:29:19,961 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb743a6f2d322b4e5 with lease ID 0xcdafecd5498cbde1: from storage DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58 node DatanodeRegistration(127.0.0.1:33973, datanodeUuid=1d187868-3ce7-467a-aaf4-09d22fe90c6d, infoPort=44531, infoSecurePort=0, ipcPort=45143, storageInfo=lv=-57;cid=testClusterID;nsid=1369661529;c=1733462937788), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:29:19,961 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb743a6f2d322b4e5 with lease ID 0xcdafecd5498cbde1: from storage DS-bc6490d7-bcfd-4b63-ba23-0fbff276b251 node DatanodeRegistration(127.0.0.1:33973, datanodeUuid=1d187868-3ce7-467a-aaf4-09d22fe90c6d, infoPort=44531, infoSecurePort=0, ipcPort=45143, storageInfo=lv=-57;cid=testClusterID;nsid=1369661529;c=1733462937788), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:29:20,084 WARN  [Thread-1432 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:29:20,087 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfce2a72b3357f6dc with lease ID 0xcdafecd5498cbde2: from storage DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af node DatanodeRegistration(127.0.0.1:33221, datanodeUuid=2976047c-a66b-4db7-8cdd-428d3778c51a, infoPort=38287, infoSecurePort=0, ipcPort=38015, storageInfo=lv=-57;cid=testClusterID;nsid=1369661529;c=1733462937788), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:29:20,087 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfce2a72b3357f6dc with lease ID 0xcdafecd5498cbde2: from storage DS-86440bd0-79b5-46b6-b1cb-24cb0d1ace18 node DatanodeRegistration(127.0.0.1:33221, datanodeUuid=2976047c-a66b-4db7-8cdd-428d3778c51a, infoPort=38287, infoSecurePort=0, ipcPort=38015, storageInfo=lv=-57;cid=testClusterID;nsid=1369661529;c=1733462937788), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:29:20,480 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-12-06T05:29:20,482 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-12-06T05:29:20,484 ERROR [FSHLog-0-hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373-prefix:ec46afd49254,38219,1733462940445 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38877,DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:20,484 WARN  [FSHLog-0-hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373-prefix:ec46afd49254,38219,1733462940445 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38877,DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:20,484 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ec46afd49254%2C38219%2C1733462940445:(num 1733462955164) roll requested
2024-12-06T05:29:20,485 INFO  [regionserver/ec46afd49254:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C38219%2C1733462940445.1733462960484
2024-12-06T05:29:20,493 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462955164 newFile=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462960484
2024-12-06T05:29:20,493 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:20,493 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:20,494 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:20,494 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:20,494 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:20,494 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462955164 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462960484
2024-12-06T05:29:20,494 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38877,DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:20,494 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38877,DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:20,494 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462955164
2024-12-06T05:29:20,495 WARN  [IPC Server handler 0 on default port 45395 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462955164 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-12-06T05:29:20,495 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462955164 after 1ms
2024-12-06T05:29:20,498 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38287:38287),(127.0.0.1/127.0.0.1:44531:44531)]
2024-12-06T05:29:20,498 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462955164 is not closed yet, will try archiving it next time
2024-12-06T05:29:20,698 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:20,873 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:21,699 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:21,874 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:22,499 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C38219%2C1733462940445.1733462962499
2024-12-06T05:29:22,506 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462960484 newFile=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462962499
2024-12-06T05:29:22,506 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:22,507 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:22,507 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:22,507 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:22,507 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:22,507 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462960484 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462962499
2024-12-06T05:29:22,509 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33973 is added to blk_1073741838_1019 (size=1264)
2024-12-06T05:29:22,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33221 is added to blk_1073741838_1019 (size=1264)
2024-12-06T05:29:22,511 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462955164 is not closed yet, will try archiving it next time
2024-12-06T05:29:22,512 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38287:38287),(127.0.0.1/127.0.0.1:44531:44531)]
2024-12-06T05:29:22,513 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462955164 is not closed yet, will try archiving it next time
2024-12-06T05:29:22,513 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462941143
2024-12-06T05:29:22,513 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462941143
2024-12-06T05:29:22,513 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462941143 after 0ms
2024-12-06T05:29:22,514 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462941143
2024-12-06T05:29:22,525 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733462942059/Put/vlen=218/seqid=0]
2024-12-06T05:29:22,526 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1733462951742/Put/vlen=1045/seqid=0]
2024-12-06T05:29:22,526 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462941143
2024-12-06T05:29:22,526 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462955164
2024-12-06T05:29:22,526 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462955164
2024-12-06T05:29:22,526 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462955164 after 0ms
2024-12-06T05:29:22,526 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462955164
2024-12-06T05:29:22,531 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1733462955164/Put/vlen=1045/seqid=0]
2024-12-06T05:29:22,531 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1733462957176/Put/vlen=1045/seqid=0]
2024-12-06T05:29:22,532 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462955164
2024-12-06T05:29:22,532 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462960484
2024-12-06T05:29:22,532 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462960484
2024-12-06T05:29:22,532 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462960484 after 0ms
2024-12-06T05:29:22,532 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462960484
2024-12-06T05:29:22,537 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1733462960483/Put/vlen=1045/seqid=0]
2024-12-06T05:29:22,537 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462962499
2024-12-06T05:29:22,537 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462962499
2024-12-06T05:29:22,537 WARN  [IPC Server handler 2 on default port 45395 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462962499 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-12-06T05:29:22,538 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462962499 after 1ms
2024-12-06T05:29:22,699 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:22,875 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:22,961 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-06T05:29:23,088 WARN  [ResponseProcessor for block BP-374932401-172.17.0.3-1733462937788:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-374932401-172.17.0.3-1733462937788:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:23,088 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1489413447_22 at /127.0.0.1:48572 [Receiving block BP-374932401-172.17.0.3-1733462937788:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:33221:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48572 dst: /127.0.0.1:33221
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33221 remote=/127.0.0.1:48572]. Total timeout mills is 60000, 59418 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:23,088 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1489413447_22 at /127.0.0.1:37028 [Receiving block BP-374932401-172.17.0.3-1733462937788:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:33973:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37028 dst: /127.0.0.1:33973
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:23,088 WARN  [DataStreamer for file /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462962499 block BP-374932401-172.17.0.3-1733462937788:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-374932401-172.17.0.3-1733462937788:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33221,DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af,DISK], DatanodeInfoWithStorage[127.0.0.1:33973,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33221,DS-027636eb-a90e-41bc-ad82-a5ca8ec5c0af,DISK]) is bad.
2024-12-06T05:29:23,090 WARN  [DataStreamer for file /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462962499 block BP-374932401-172.17.0.3-1733462937788:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-374932401-172.17.0.3-1733462937788:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:23,095 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33221 is added to blk_1073741839_1022 (size=85)
2024-12-06T05:29:23,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33973 is added to blk_1073741839_1022 (size=85)
2024-12-06T05:29:23,700 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:23,876 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:24,497 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462955164 after 4003ms
2024-12-06T05:29:24,701 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:24,876 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:25,702 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:25,877 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:26,539 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462962499 after 4002ms
2024-12-06T05:29:26,539 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462962499
2024-12-06T05:29:26,553 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462962499
2024-12-06T05:29:26,553 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 5b3a8be16cfff4418b5ccc392be46d27 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-12-06T05:29:26,554 ERROR [FSHLog-0-hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373-prefix:ec46afd49254,38219,1733462940445 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-374932401-172.17.0.3-1733462937788:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:26,555 WARN  [FSHLog-0-hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373-prefix:ec46afd49254,38219,1733462940445 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-374932401-172.17.0.3-1733462937788:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:26,555 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ec46afd49254%2C38219%2C1733462940445:(num 1733462962499) roll requested
2024-12-06T05:29:26,556 INFO  [regionserver/ec46afd49254:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C38219%2C1733462940445.1733462966555
2024-12-06T05:29:26,563 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462962499 newFile=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462966555
2024-12-06T05:29:26,564 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:26,564 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:26,564 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:26,564 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:26,564 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:26,564 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462962499 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462966555
2024-12-06T05:29:26,564 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-374932401-172.17.0.3-1733462937788:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:26,565 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-374932401-172.17.0.3-1733462937788:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:26,565 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462962499
2024-12-06T05:29:26,565 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462962499 after 0ms
2024-12-06T05:29:26,566 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.1733462962499 to hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/oldWALs/ec46afd49254%2C38219%2C1733462940445.1733462962499
2024-12-06T05:29:26,567 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38287:38287),(127.0.0.1/127.0.0.1:44531:44531)]
2024-12-06T05:29:26,584 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/default/TestLogRolling-testLogRollOnPipelineRestart/5b3a8be16cfff4418b5ccc392be46d27/.tmp/info/ee54b9ff244a454a819ed6bd389303ec is 1080, key is row1002/info:/1733462951742/Put/seqid=0
2024-12-06T05:29:26,590 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33973 is added to blk_1073741841_1024 (size=9270)
2024-12-06T05:29:26,590 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33221 is added to blk_1073741841_1024 (size=9270)
2024-12-06T05:29:26,591 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/default/TestLogRolling-testLogRollOnPipelineRestart/5b3a8be16cfff4418b5ccc392be46d27/.tmp/info/ee54b9ff244a454a819ed6bd389303ec
2024-12-06T05:29:26,598 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/default/TestLogRolling-testLogRollOnPipelineRestart/5b3a8be16cfff4418b5ccc392be46d27/.tmp/info/ee54b9ff244a454a819ed6bd389303ec as hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/default/TestLogRolling-testLogRollOnPipelineRestart/5b3a8be16cfff4418b5ccc392be46d27/info/ee54b9ff244a454a819ed6bd389303ec
2024-12-06T05:29:26,607 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/default/TestLogRolling-testLogRollOnPipelineRestart/5b3a8be16cfff4418b5ccc392be46d27/info/ee54b9ff244a454a819ed6bd389303ec, entries=4, sequenceid=8, filesize=9.1 K
2024-12-06T05:29:26,608 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 5b3a8be16cfff4418b5ccc392be46d27 in 55ms, sequenceid=8, compaction requested=false
2024-12-06T05:29:26,608 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5b3a8be16cfff4418b5ccc392be46d27:

2024-12-06T05:29:26,608 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-12-06T05:29:26,608 ERROR [FSHLog-0-hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373-prefix:ec46afd49254,38219,1733462940445.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33335,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:26,609 WARN  [FSHLog-0-hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373-prefix:ec46afd49254,38219,1733462940445.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33335,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:26,609 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ec46afd49254%2C38219%2C1733462940445.meta:.meta(num 1733462941517) roll requested
2024-12-06T05:29:26,609 INFO  [regionserver/ec46afd49254:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C38219%2C1733462940445.meta.1733462966609.meta
2024-12-06T05:29:26,615 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:26,615 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:26,615 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:26,615 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:26,615 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:26,615 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.meta.1733462941517.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.meta.1733462966609.meta
2024-12-06T05:29:26,616 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33335,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:26,616 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33335,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:26,616 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.meta.1733462941517.meta
2024-12-06T05:29:26,616 WARN  [IPC Server handler 0 on default port 45395 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.meta.1733462941517.meta has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741834_1014
2024-12-06T05:29:26,616 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.meta.1733462941517.meta after 0ms
2024-12-06T05:29:26,618 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38287:38287),(127.0.0.1/127.0.0.1:44531:44531)]
2024-12-06T05:29:26,618 DEBUG [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.meta.1733462941517.meta is not closed yet, will try archiving it next time
2024-12-06T05:29:26,632 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740/.tmp/info/04171652e3bd4af68630482a4ad35612 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27./info:regioninfo/1733462942063/Put/seqid=0
2024-12-06T05:29:26,637 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33973 is added to blk_1073741843_1027 (size=7125)
2024-12-06T05:29:26,637 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33221 is added to blk_1073741843_1027 (size=7125)
2024-12-06T05:29:26,638 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740/.tmp/info/04171652e3bd4af68630482a4ad35612
2024-12-06T05:29:26,656 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740/.tmp/ns/0bf8d08e136142d8b2898e7ef0c1e774 is 43, key is default/ns:d/1733462941616/Put/seqid=0
2024-12-06T05:29:26,660 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33221 is added to blk_1073741844_1028 (size=5153)
2024-12-06T05:29:26,660 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33973 is added to blk_1073741844_1028 (size=5153)
2024-12-06T05:29:26,661 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740/.tmp/ns/0bf8d08e136142d8b2898e7ef0c1e774
2024-12-06T05:29:26,684 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740/.tmp/table/6090137b30874413b018ee49ba09eb60 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1733462942074/Put/seqid=0
2024-12-06T05:29:26,689 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33221 is added to blk_1073741845_1029 (size=5438)
2024-12-06T05:29:26,689 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33973 is added to blk_1073741845_1029 (size=5438)
2024-12-06T05:29:26,690 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740/.tmp/table/6090137b30874413b018ee49ba09eb60
2024-12-06T05:29:26,696 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740/.tmp/info/04171652e3bd4af68630482a4ad35612 as hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740/info/04171652e3bd4af68630482a4ad35612
2024-12-06T05:29:26,702 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740/info/04171652e3bd4af68630482a4ad35612, entries=10, sequenceid=11, filesize=7.0 K
2024-12-06T05:29:26,702 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740/.tmp/ns/0bf8d08e136142d8b2898e7ef0c1e774 as hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740/ns/0bf8d08e136142d8b2898e7ef0c1e774
2024-12-06T05:29:26,703 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:26,708 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740/ns/0bf8d08e136142d8b2898e7ef0c1e774, entries=2, sequenceid=11, filesize=5.0 K
2024-12-06T05:29:26,709 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740/.tmp/table/6090137b30874413b018ee49ba09eb60 as hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740/table/6090137b30874413b018ee49ba09eb60
2024-12-06T05:29:26,715 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740/table/6090137b30874413b018ee49ba09eb60, entries=2, sequenceid=11, filesize=5.3 K
2024-12-06T05:29:26,716 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 108ms, sequenceid=11, compaction requested=false
2024-12-06T05:29:26,716 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-06T05:29:26,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-06T05:29:26,721 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T05:29:26,721 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:29:26,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:29:26,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:29:26,721 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-06T05:29:26,721 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-06T05:29:26,721 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=364509557, stopped=false
2024-12-06T05:29:26,721 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ec46afd49254,43165,1733462940279
2024-12-06T05:29:26,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T05:29:26,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T05:29:26,742 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T05:29:26,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:26,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:26,742 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T05:29:26,742 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:29:26,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:29:26,743 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ec46afd49254,38219,1733462940445' *****
2024-12-06T05:29:26,743 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-06T05:29:26,743 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:29:26,743 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:29:26,743 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-06T05:29:26,743 INFO  [RS:0;ec46afd49254:38219 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-06T05:29:26,743 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-06T05:29:26,743 INFO  [RS:0;ec46afd49254:38219 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-06T05:29:26,743 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.HRegionServer(3091): Received CLOSE for 5b3a8be16cfff4418b5ccc392be46d27
2024-12-06T05:29:26,743 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.HRegionServer(959): stopping server ec46afd49254,38219,1733462940445
2024-12-06T05:29:26,743 INFO  [RS:0;ec46afd49254:38219 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T05:29:26,743 INFO  [RS:0;ec46afd49254:38219 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ec46afd49254:38219.
2024-12-06T05:29:26,743 DEBUG [RS:0;ec46afd49254:38219 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:29:26,743 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5b3a8be16cfff4418b5ccc392be46d27, disabling compactions & flushes
2024-12-06T05:29:26,744 DEBUG [RS:0;ec46afd49254:38219 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:29:26,744 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.
2024-12-06T05:29:26,744 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.
2024-12-06T05:29:26,744 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-06T05:29:26,744 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27. after waiting 0 ms
2024-12-06T05:29:26,744 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-06T05:29:26,744 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-06T05:29:26,744 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.
2024-12-06T05:29:26,744 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-06T05:29:26,744 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-06T05:29:26,744 DEBUG [RS:0;ec46afd49254:38219 {}] regionserver.HRegionServer(1325): Online Regions={5b3a8be16cfff4418b5ccc392be46d27=TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27., 1588230740=hbase:meta,,1.1588230740}
2024-12-06T05:29:26,744 DEBUG [RS:0;ec46afd49254:38219 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5b3a8be16cfff4418b5ccc392be46d27
2024-12-06T05:29:26,744 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T05:29:26,744 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T05:29:26,744 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T05:29:26,744 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T05:29:26,744 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T05:29:26,748 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/default/TestLogRolling-testLogRollOnPipelineRestart/5b3a8be16cfff4418b5ccc392be46d27/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-12-06T05:29:26,748 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-06T05:29:26,748 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.
2024-12-06T05:29:26,748 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T05:29:26,748 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5b3a8be16cfff4418b5ccc392be46d27:
Waiting for close lock at 1733462966743Running coprocessor pre-close hooks at 1733462966743Disabling compacts and flushes for region at 1733462966743Disabling writes for close at 1733462966744 (+1 ms)Writing region close event to WAL at 1733462966744Running coprocessor post-close hooks at 1733462966748 (+4 ms)Closed at 1733462966748
2024-12-06T05:29:26,748 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T05:29:26,748 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733462966744Running coprocessor pre-close hooks at 1733462966744Disabling compacts and flushes for region at 1733462966744Disabling writes for close at 1733462966744Writing region close event to WAL at 1733462966745 (+1 ms)Running coprocessor post-close hooks at 1733462966748 (+3 ms)Closed at 1733462966748
2024-12-06T05:29:26,748 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733462941693.5b3a8be16cfff4418b5ccc392be46d27.
2024-12-06T05:29:26,748 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-06T05:29:26,878 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:26,944 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.HRegionServer(976): stopping server ec46afd49254,38219,1733462940445; all regions closed.
2024-12-06T05:29:26,945 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:26,945 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:26,945 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:26,946 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:26,946 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:26,948 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33973 is added to blk_1073741842_1025 (size=825)
2024-12-06T05:29:26,948 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33221 is added to blk_1073741842_1025 (size=825)
2024-12-06T05:29:27,008 INFO  [regionserver/ec46afd49254:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-06T05:29:27,008 INFO  [regionserver/ec46afd49254:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-06T05:29:27,010 INFO  [regionserver/ec46afd49254:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T05:29:27,703 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:27,879 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:28,704 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:28,880 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:29,089 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1014: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-06T05:29:29,705 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:29,880 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:30,256 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T05:29:30,618 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.meta.1733462941517.meta after 4002ms
2024-12-06T05:29:30,619 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/WALs/ec46afd49254,38219,1733462940445/ec46afd49254%2C38219%2C1733462940445.meta.1733462941517.meta to hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/oldWALs/ec46afd49254%2C38219%2C1733462940445.meta.1733462941517.meta
2024-12-06T05:29:30,626 DEBUG [RS:0;ec46afd49254:38219 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/oldWALs
2024-12-06T05:29:30,627 INFO  [RS:0;ec46afd49254:38219 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ec46afd49254%2C38219%2C1733462940445.meta:.meta(num 1733462966609)
2024-12-06T05:29:30,627 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:30,627 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:30,628 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:30,628 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:30,628 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:30,630 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33973 is added to blk_1073741840_1023 (size=1162)
2024-12-06T05:29:30,630 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33221 is added to blk_1073741840_1023 (size=1162)
2024-12-06T05:29:30,634 DEBUG [RS:0;ec46afd49254:38219 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/oldWALs
2024-12-06T05:29:30,634 INFO  [RS:0;ec46afd49254:38219 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ec46afd49254%2C38219%2C1733462940445:(num 1733462966555)
2024-12-06T05:29:30,634 DEBUG [RS:0;ec46afd49254:38219 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:29:30,634 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T05:29:30,634 INFO  [RS:0;ec46afd49254:38219 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T05:29:30,635 INFO  [RS:0;ec46afd49254:38219 {}] hbase.ChoreService(370): Chore service for: regionserver/ec46afd49254:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-06T05:29:30,635 INFO  [RS:0;ec46afd49254:38219 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T05:29:30,635 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T05:29:30,635 INFO  [RS:0;ec46afd49254:38219 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38219
2024-12-06T05:29:30,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec46afd49254,38219,1733462940445
2024-12-06T05:29:30,700 INFO  [RS:0;ec46afd49254:38219 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T05:29:30,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T05:29:30,706 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:30,711 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec46afd49254,38219,1733462940445]
2024-12-06T05:29:30,721 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ec46afd49254,38219,1733462940445 already deleted, retry=false
2024-12-06T05:29:30,722 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ec46afd49254,38219,1733462940445 expired; onlineServers=0
2024-12-06T05:29:30,722 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ec46afd49254,43165,1733462940279' *****
2024-12-06T05:29:30,722 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-06T05:29:30,722 INFO  [M:0;ec46afd49254:43165 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T05:29:30,722 INFO  [M:0;ec46afd49254:43165 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T05:29:30,722 DEBUG [M:0;ec46afd49254:43165 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-06T05:29:30,722 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-06T05:29:30,722 DEBUG [M:0;ec46afd49254:43165 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-06T05:29:30,722 DEBUG [master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462940794 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462940794,5,FailOnTimeoutGroup]
2024-12-06T05:29:30,722 DEBUG [master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462940794 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462940794,5,FailOnTimeoutGroup]
2024-12-06T05:29:30,723 INFO  [M:0;ec46afd49254:43165 {}] hbase.ChoreService(370): Chore service for: master/ec46afd49254:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-06T05:29:30,723 INFO  [M:0;ec46afd49254:43165 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T05:29:30,723 DEBUG [M:0;ec46afd49254:43165 {}] master.HMaster(1795): Stopping service threads
2024-12-06T05:29:30,723 INFO  [M:0;ec46afd49254:43165 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-06T05:29:30,723 INFO  [M:0;ec46afd49254:43165 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T05:29:30,723 INFO  [M:0;ec46afd49254:43165 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-06T05:29:30,723 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-06T05:29:30,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-06T05:29:30,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:30,732 DEBUG [M:0;ec46afd49254:43165 {}] zookeeper.ZKUtil(347): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-06T05:29:30,732 WARN  [M:0;ec46afd49254:43165 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-06T05:29:30,733 INFO  [M:0;ec46afd49254:43165 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/.lastflushedseqids
2024-12-06T05:29:30,742 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33973 is added to blk_1073741846_1030 (size=130)
2024-12-06T05:29:30,742 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33221 is added to blk_1073741846_1030 (size=130)
2024-12-06T05:29:30,743 INFO  [M:0;ec46afd49254:43165 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-06T05:29:30,743 INFO  [M:0;ec46afd49254:43165 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-06T05:29:30,743 DEBUG [M:0;ec46afd49254:43165 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T05:29:30,743 INFO  [M:0;ec46afd49254:43165 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:29:30,743 DEBUG [M:0;ec46afd49254:43165 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:29:30,743 DEBUG [M:0;ec46afd49254:43165 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T05:29:30,743 DEBUG [M:0;ec46afd49254:43165 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:29:30,743 INFO  [M:0;ec46afd49254:43165 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB
2024-12-06T05:29:30,744 ERROR [FSHLog-0-hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData-prefix:ec46afd49254,43165,1733462940279 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33335,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:30,744 WARN  [FSHLog-0-hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData-prefix:ec46afd49254,43165,1733462940279 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33335,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:30,744 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog ec46afd49254%2C43165%2C1733462940279:(num 1733462940585) roll requested
2024-12-06T05:29:30,744 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C43165%2C1733462940279.1733462970744
2024-12-06T05:29:30,748 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:30,748 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:30,748 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:30,749 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:30,749 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:30,749 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/WALs/ec46afd49254,43165,1733462940279/ec46afd49254%2C43165%2C1733462940279.1733462940585 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/WALs/ec46afd49254,43165,1733462940279/ec46afd49254%2C43165%2C1733462940279.1733462970744
2024-12-06T05:29:30,749 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33335,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:30,749 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33335,DS-ab6cc735-559d-40ca-aa7f-6fd36fad8f58,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T05:29:30,749 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/WALs/ec46afd49254,43165,1733462940279/ec46afd49254%2C43165%2C1733462940279.1733462940585
2024-12-06T05:29:30,749 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38287:38287),(127.0.0.1/127.0.0.1:44531:44531)]
2024-12-06T05:29:30,750 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/WALs/ec46afd49254,43165,1733462940279/ec46afd49254%2C43165%2C1733462940279.1733462940585 is not closed yet, will try archiving it next time
2024-12-06T05:29:30,750 WARN  [IPC Server handler 1 on default port 45395 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/WALs/ec46afd49254,43165,1733462940279/ec46afd49254%2C43165%2C1733462940279.1733462940585 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015
2024-12-06T05:29:30,750 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/WALs/ec46afd49254,43165,1733462940279/ec46afd49254%2C43165%2C1733462940279.1733462940585 after 1ms
2024-12-06T05:29:30,763 DEBUG [M:0;ec46afd49254:43165 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d5c132a9034d4b109d947e333ed57a40 is 82, key is hbase:meta,,1/info:regioninfo/1733462941544/Put/seqid=0
2024-12-06T05:29:30,768 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33221 is added to blk_1073741848_1033 (size=5672)
2024-12-06T05:29:30,768 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33973 is added to blk_1073741848_1033 (size=5672)
2024-12-06T05:29:30,768 INFO  [M:0;ec46afd49254:43165 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d5c132a9034d4b109d947e333ed57a40
2024-12-06T05:29:30,787 DEBUG [M:0;ec46afd49254:43165 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8ae9aad0c38f4b8f86ed3959a3b69190 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733462942079/Put/seqid=0
2024-12-06T05:29:30,792 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33221 is added to blk_1073741849_1034 (size=6118)
2024-12-06T05:29:30,792 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33973 is added to blk_1073741849_1034 (size=6118)
2024-12-06T05:29:30,792 INFO  [M:0;ec46afd49254:43165 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8ae9aad0c38f4b8f86ed3959a3b69190
2024-12-06T05:29:30,811 DEBUG [M:0;ec46afd49254:43165 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/489b470c50f6432b8fe9d05fdd280c36 is 69, key is ec46afd49254,38219,1733462940445/rs:state/1733462940905/Put/seqid=0
2024-12-06T05:29:30,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:29:30,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38219-0x101a91d560e0001, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:29:30,811 INFO  [RS:0;ec46afd49254:38219 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T05:29:30,811 INFO  [RS:0;ec46afd49254:38219 {}] regionserver.HRegionServer(1031): Exiting; stopping=ec46afd49254,38219,1733462940445; zookeeper connection closed.
2024-12-06T05:29:30,812 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3edaf681 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3edaf681
2024-12-06T05:29:30,812 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-06T05:29:30,816 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33221 is added to blk_1073741850_1035 (size=5156)
2024-12-06T05:29:30,816 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33973 is added to blk_1073741850_1035 (size=5156)
2024-12-06T05:29:30,816 INFO  [M:0;ec46afd49254:43165 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/489b470c50f6432b8fe9d05fdd280c36
2024-12-06T05:29:30,834 DEBUG [M:0;ec46afd49254:43165 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/963a7a59c51b40019e2ce42959921f57 is 52, key is load_balancer_on/state:d/1733462941687/Put/seqid=0
2024-12-06T05:29:30,839 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33973 is added to blk_1073741851_1036 (size=5056)
2024-12-06T05:29:30,839 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33221 is added to blk_1073741851_1036 (size=5056)
2024-12-06T05:29:30,840 INFO  [M:0;ec46afd49254:43165 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/963a7a59c51b40019e2ce42959921f57
2024-12-06T05:29:30,845 DEBUG [M:0;ec46afd49254:43165 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d5c132a9034d4b109d947e333ed57a40 as hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d5c132a9034d4b109d947e333ed57a40
2024-12-06T05:29:30,850 INFO  [M:0;ec46afd49254:43165 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d5c132a9034d4b109d947e333ed57a40, entries=8, sequenceid=56, filesize=5.5 K
2024-12-06T05:29:30,851 DEBUG [M:0;ec46afd49254:43165 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8ae9aad0c38f4b8f86ed3959a3b69190 as hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8ae9aad0c38f4b8f86ed3959a3b69190
2024-12-06T05:29:30,856 INFO  [M:0;ec46afd49254:43165 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8ae9aad0c38f4b8f86ed3959a3b69190, entries=6, sequenceid=56, filesize=6.0 K
2024-12-06T05:29:30,857 DEBUG [M:0;ec46afd49254:43165 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/489b470c50f6432b8fe9d05fdd280c36 as hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/489b470c50f6432b8fe9d05fdd280c36
2024-12-06T05:29:30,862 INFO  [M:0;ec46afd49254:43165 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/489b470c50f6432b8fe9d05fdd280c36, entries=1, sequenceid=56, filesize=5.0 K
2024-12-06T05:29:30,863 DEBUG [M:0;ec46afd49254:43165 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/963a7a59c51b40019e2ce42959921f57 as hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/963a7a59c51b40019e2ce42959921f57
2024-12-06T05:29:30,868 INFO  [M:0;ec46afd49254:43165 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/963a7a59c51b40019e2ce42959921f57, entries=1, sequenceid=56, filesize=4.9 K
2024-12-06T05:29:30,869 INFO  [M:0;ec46afd49254:43165 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 126ms, sequenceid=56, compaction requested=false
2024-12-06T05:29:30,872 INFO  [M:0;ec46afd49254:43165 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:29:30,872 DEBUG [M:0;ec46afd49254:43165 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733462970743Disabling compacts and flushes for region at 1733462970743Disabling writes for close at 1733462970743Obtaining lock to block concurrent updates at 1733462970743Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733462970743Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1733462970744 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733462970750 (+6 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733462970750Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733462970763 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733462970763Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733462970773 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733462970786 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733462970786Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733462970796 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733462970810 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733462970810Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733462970822 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733462970834 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733462970834Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c2a007: reopening flushed file at 1733462970844 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b76b999: reopening flushed file at 1733462970850 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cee5a27: reopening flushed file at 1733462970856 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6115769a: reopening flushed file at 1733462970862 (+6 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 126ms, sequenceid=56, compaction requested=false at 1733462970869 (+7 ms)Writing region close event to WAL at 1733462970872 (+3 ms)Closed at 1733462970872
2024-12-06T05:29:30,873 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:30,873 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:30,873 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:30,873 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:30,873 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:29:30,875 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33973 is added to blk_1073741847_1031 (size=757)
2024-12-06T05:29:30,875 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33221 is added to blk_1073741847_1031 (size=757)
2024-12-06T05:29:30,881 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:31,706 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:31,752 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:31,753 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:31,769 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:31,769 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:31,769 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:31,770 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:31,770 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:31,770 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:31,772 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:31,772 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:31,772 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:31,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:31,776 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:31,776 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:31,882 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:32,088 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-06T05:29:32,280 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T05:29:32,285 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:32,285 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:32,286 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:32,286 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:32,300 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:32,301 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:32,301 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:32,301 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:32,301 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:32,301 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:32,304 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:32,304 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:32,304 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:32,306 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:32,708 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:32,883 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:33,709 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:33,884 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:34,710 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:34,752 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/WALs/ec46afd49254,43165,1733462940279/ec46afd49254%2C43165%2C1733462940279.1733462940585 after 4002ms
2024-12-06T05:29:34,753 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/WALs/ec46afd49254,43165,1733462940279/ec46afd49254%2C43165%2C1733462940279.1733462940585 to hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/oldWALs/ec46afd49254%2C43165%2C1733462940279.1733462940585
2024-12-06T05:29:34,760 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/MasterData/oldWALs/ec46afd49254%2C43165%2C1733462940279.1733462940585 to hdfs://localhost:45395/user/jenkins/test-data/5530523f-1b8a-30f3-df5e-e02ad2dc8373/oldWALs/ec46afd49254%2C43165%2C1733462940279.1733462940585$masterlocalwal$
2024-12-06T05:29:34,760 INFO  [M:0;ec46afd49254:43165 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-06T05:29:34,760 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T05:29:34,760 INFO  [M:0;ec46afd49254:43165 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43165
2024-12-06T05:29:34,761 INFO  [M:0;ec46afd49254:43165 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T05:29:34,885 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:34,980 INFO  [M:0;ec46afd49254:43165 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T05:29:34,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:29:34,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x101a91d560e0000, quorum=127.0.0.1:63652, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:29:34,982 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@39e17fd3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:29:34,983 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5b16a6e2{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:29:34,983 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:29:34,983 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fc558ce{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:29:34,983 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@160f7ba9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.log.dir/,STOPPED}
2024-12-06T05:29:34,984 WARN  [BP-374932401-172.17.0.3-1733462937788 heartbeating to localhost/127.0.0.1:45395 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:29:34,984 WARN  [BP-374932401-172.17.0.3-1733462937788 heartbeating to localhost/127.0.0.1:45395 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-374932401-172.17.0.3-1733462937788 (Datanode Uuid 2976047c-a66b-4db7-8cdd-428d3778c51a) service to localhost/127.0.0.1:45395
2024-12-06T05:29:34,984 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:29:34,984 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:29:34,985 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/data/data3/current/BP-374932401-172.17.0.3-1733462937788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:29:34,985 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/data/data4/current/BP-374932401-172.17.0.3-1733462937788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:29:34,986 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:29:34,989 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@48919a97{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:29:34,989 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4778d192{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:29:34,989 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:29:34,989 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d237fac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:29:34,990 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a79eeb5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.log.dir/,STOPPED}
2024-12-06T05:29:34,991 WARN  [BP-374932401-172.17.0.3-1733462937788 heartbeating to localhost/127.0.0.1:45395 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:29:34,991 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:29:34,991 WARN  [BP-374932401-172.17.0.3-1733462937788 heartbeating to localhost/127.0.0.1:45395 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-374932401-172.17.0.3-1733462937788 (Datanode Uuid 1d187868-3ce7-467a-aaf4-09d22fe90c6d) service to localhost/127.0.0.1:45395
2024-12-06T05:29:34,991 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:29:34,992 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/data/data1/current/BP-374932401-172.17.0.3-1733462937788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:29:34,992 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/cluster_7255091c-1a3c-db93-cc69-f1289900fd75/data/data2/current/BP-374932401-172.17.0.3-1733462937788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:29:34,992 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:29:34,997 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@586e8021{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T05:29:34,998 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7ec5b9af{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:29:34,998 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:29:34,998 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11255fea{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:29:34,998 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@61f9169f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.log.dir/,STOPPED}
2024-12-06T05:29:35,005 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-06T05:29:35,024 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-06T05:29:35,031 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=180 (was 155)
Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45395
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45395
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45395 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45395 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45395
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:45395
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45395 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:45395
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=457 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=230 (was 233), ProcessCount=11 (was 11), AvailableMemoryMB=7095 (was 6962) - AvailableMemoryMB LEAK? -
2024-12-06T05:29:35,038 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=180, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=230, ProcessCount=11, AvailableMemoryMB=7095
2024-12-06T05:29:35,038 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-06T05:29:35,039 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.log.dir so I do NOT create it in target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536
2024-12-06T05:29:35,039 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/683ec0dc-9d89-05e4-1562-1feff1548ecc/hadoop.tmp.dir so I do NOT create it in target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536
2024-12-06T05:29:35,039 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/cluster_b8ebfc14-66ba-4c28-dc87-1aa08dd53a02, deleteOnExit=true
2024-12-06T05:29:35,039 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-06T05:29:35,039 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/test.cache.data in system properties and HBase conf
2024-12-06T05:29:35,039 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/hadoop.tmp.dir in system properties and HBase conf
2024-12-06T05:29:35,039 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/hadoop.log.dir in system properties and HBase conf
2024-12-06T05:29:35,039 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-06T05:29:35,039 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-06T05:29:35,039 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-06T05:29:35,040 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-06T05:29:35,040 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-06T05:29:35,040 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-06T05:29:35,040 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-06T05:29:35,040 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T05:29:35,040 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-06T05:29:35,040 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-06T05:29:35,040 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T05:29:35,040 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T05:29:35,040 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-06T05:29:35,040 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/nfs.dump.dir in system properties and HBase conf
2024-12-06T05:29:35,040 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/java.io.tmpdir in system properties and HBase conf
2024-12-06T05:29:35,041 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T05:29:35,041 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-06T05:29:35,041 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-06T05:29:35,051 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T05:29:35,445 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:29:35,449 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:29:35,450 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:29:35,450 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:29:35,450 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T05:29:35,450 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:29:35,451 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46a495b0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:29:35,451 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4cf5e3df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:29:35,539 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1b5fa12b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/java.io.tmpdir/jetty-localhost-46769-hadoop-hdfs-3_4_1-tests_jar-_-any-5638297703530735224/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T05:29:35,540 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10128232{HTTP/1.1, (http/1.1)}{localhost:46769}
2024-12-06T05:29:35,540 INFO  [Time-limited test {}] server.Server(415): Started @195151ms
2024-12-06T05:29:35,550 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T05:29:35,711 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:35,789 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:29:35,792 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:29:35,793 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:29:35,793 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:29:35,793 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T05:29:35,794 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69ff6367{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:29:35,794 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4911f4c5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:29:35,885 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:35,896 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@10fa3dab{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/java.io.tmpdir/jetty-localhost-41509-hadoop-hdfs-3_4_1-tests_jar-_-any-16514537357247619913/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:29:35,897 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@bc64a6a{HTTP/1.1, (http/1.1)}{localhost:41509}
2024-12-06T05:29:35,897 INFO  [Time-limited test {}] server.Server(415): Started @195508ms
2024-12-06T05:29:35,898 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:29:35,922 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:29:35,924 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:29:35,925 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:29:35,925 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:29:35,925 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T05:29:35,926 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3dca8de9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:29:35,926 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6623a859{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:29:36,016 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4e363d0c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/java.io.tmpdir/jetty-localhost-40603-hadoop-hdfs-3_4_1-tests_jar-_-any-11436187803624406466/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:29:36,017 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5daea1f2{HTTP/1.1, (http/1.1)}{localhost:40603}
2024-12-06T05:29:36,017 INFO  [Time-limited test {}] server.Server(415): Started @195628ms
2024-12-06T05:29:36,018 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:29:36,505 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T05:29:36,505 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-06T05:29:36,506 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-06T05:29:36,506 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-06T05:29:36,712 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:36,886 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:37,252 WARN  [Thread-1653 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/cluster_b8ebfc14-66ba-4c28-dc87-1aa08dd53a02/data/data2/current/BP-1266242495-172.17.0.3-1733462975061/current, will proceed with Du for space computation calculation, 
2024-12-06T05:29:37,252 WARN  [Thread-1652 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/cluster_b8ebfc14-66ba-4c28-dc87-1aa08dd53a02/data/data1/current/BP-1266242495-172.17.0.3-1733462975061/current, will proceed with Du for space computation calculation, 
2024-12-06T05:29:37,266 WARN  [Thread-1616 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:29:37,268 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfd3e983c6f88d08d with lease ID 0x70a6e1bcf93da130: Processing first storage report for DS-1a4defd2-b66c-45f7-9f25-482b0767ae99 from datanode DatanodeRegistration(127.0.0.1:35385, datanodeUuid=a8a1f2d0-1b84-418d-8220-695b757af27f, infoPort=35265, infoSecurePort=0, ipcPort=39657, storageInfo=lv=-57;cid=testClusterID;nsid=574894304;c=1733462975061)
2024-12-06T05:29:37,268 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfd3e983c6f88d08d with lease ID 0x70a6e1bcf93da130: from storage DS-1a4defd2-b66c-45f7-9f25-482b0767ae99 node DatanodeRegistration(127.0.0.1:35385, datanodeUuid=a8a1f2d0-1b84-418d-8220-695b757af27f, infoPort=35265, infoSecurePort=0, ipcPort=39657, storageInfo=lv=-57;cid=testClusterID;nsid=574894304;c=1733462975061), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:29:37,268 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfd3e983c6f88d08d with lease ID 0x70a6e1bcf93da130: Processing first storage report for DS-58efe748-872c-4c74-ae45-10669ac50038 from datanode DatanodeRegistration(127.0.0.1:35385, datanodeUuid=a8a1f2d0-1b84-418d-8220-695b757af27f, infoPort=35265, infoSecurePort=0, ipcPort=39657, storageInfo=lv=-57;cid=testClusterID;nsid=574894304;c=1733462975061)
2024-12-06T05:29:37,268 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfd3e983c6f88d08d with lease ID 0x70a6e1bcf93da130: from storage DS-58efe748-872c-4c74-ae45-10669ac50038 node DatanodeRegistration(127.0.0.1:35385, datanodeUuid=a8a1f2d0-1b84-418d-8220-695b757af27f, infoPort=35265, infoSecurePort=0, ipcPort=39657, storageInfo=lv=-57;cid=testClusterID;nsid=574894304;c=1733462975061), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:29:37,488 WARN  [Thread-1663 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/cluster_b8ebfc14-66ba-4c28-dc87-1aa08dd53a02/data/data3/current/BP-1266242495-172.17.0.3-1733462975061/current, will proceed with Du for space computation calculation, 
2024-12-06T05:29:37,488 WARN  [Thread-1664 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/cluster_b8ebfc14-66ba-4c28-dc87-1aa08dd53a02/data/data4/current/BP-1266242495-172.17.0.3-1733462975061/current, will proceed with Du for space computation calculation, 
2024-12-06T05:29:37,511 WARN  [Thread-1639 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:29:37,513 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe4ccb5a2b8169f9d with lease ID 0x70a6e1bcf93da131: Processing first storage report for DS-0b48f2ff-51cb-4324-9957-ce5e71dc2b42 from datanode DatanodeRegistration(127.0.0.1:46681, datanodeUuid=d5890785-ece2-4fa7-b8b3-c610ba90c834, infoPort=42051, infoSecurePort=0, ipcPort=46547, storageInfo=lv=-57;cid=testClusterID;nsid=574894304;c=1733462975061)
2024-12-06T05:29:37,513 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe4ccb5a2b8169f9d with lease ID 0x70a6e1bcf93da131: from storage DS-0b48f2ff-51cb-4324-9957-ce5e71dc2b42 node DatanodeRegistration(127.0.0.1:46681, datanodeUuid=d5890785-ece2-4fa7-b8b3-c610ba90c834, infoPort=42051, infoSecurePort=0, ipcPort=46547, storageInfo=lv=-57;cid=testClusterID;nsid=574894304;c=1733462975061), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:29:37,513 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe4ccb5a2b8169f9d with lease ID 0x70a6e1bcf93da131: Processing first storage report for DS-3bdcbc09-e69a-41d6-9e65-4ea94a0c7283 from datanode DatanodeRegistration(127.0.0.1:46681, datanodeUuid=d5890785-ece2-4fa7-b8b3-c610ba90c834, infoPort=42051, infoSecurePort=0, ipcPort=46547, storageInfo=lv=-57;cid=testClusterID;nsid=574894304;c=1733462975061)
2024-12-06T05:29:37,513 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe4ccb5a2b8169f9d with lease ID 0x70a6e1bcf93da131: from storage DS-3bdcbc09-e69a-41d6-9e65-4ea94a0c7283 node DatanodeRegistration(127.0.0.1:46681, datanodeUuid=d5890785-ece2-4fa7-b8b3-c610ba90c834, infoPort=42051, infoSecurePort=0, ipcPort=46547, storageInfo=lv=-57;cid=testClusterID;nsid=574894304;c=1733462975061), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:29:37,547 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536
2024-12-06T05:29:37,551 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/cluster_b8ebfc14-66ba-4c28-dc87-1aa08dd53a02/zookeeper_0, clientPort=64747, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/cluster_b8ebfc14-66ba-4c28-dc87-1aa08dd53a02/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/cluster_b8ebfc14-66ba-4c28-dc87-1aa08dd53a02/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-06T05:29:37,553 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64747
2024-12-06T05:29:37,553 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:29:37,555 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:29:37,565 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741825_1001 (size=7)
2024-12-06T05:29:37,565 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741825_1001 (size=7)
2024-12-06T05:29:37,566 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040 with version=8
2024-12-06T05:29:37,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/hbase-staging
2024-12-06T05:29:37,568 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/ec46afd49254:0 server-side Connection retries=45
2024-12-06T05:29:37,568 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:29:37,568 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T05:29:37,568 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T05:29:37,568 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:29:37,568 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T05:29:37,568 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-06T05:29:37,569 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T05:29:37,569 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43729
2024-12-06T05:29:37,570 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43729 connecting to ZooKeeper ensemble=127.0.0.1:64747
2024-12-06T05:29:37,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:437290x0, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T05:29:37,683 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43729-0x101a91de7b90000 connected
2024-12-06T05:29:37,712 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:37,802 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:29:37,805 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:29:37,808 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:29:37,808 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040, hbase.cluster.distributed=false
2024-12-06T05:29:37,810 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T05:29:37,810 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43729
2024-12-06T05:29:37,810 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43729
2024-12-06T05:29:37,810 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43729
2024-12-06T05:29:37,811 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43729
2024-12-06T05:29:37,811 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43729
2024-12-06T05:29:37,824 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/ec46afd49254:0 server-side Connection retries=45
2024-12-06T05:29:37,824 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:29:37,824 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T05:29:37,824 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T05:29:37,824 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:29:37,824 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T05:29:37,824 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-06T05:29:37,824 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T05:29:37,825 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34703
2024-12-06T05:29:37,825 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34703 connecting to ZooKeeper ensemble=127.0.0.1:64747
2024-12-06T05:29:37,826 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:29:37,827 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:29:37,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:347030x0, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T05:29:37,833 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:29:37,833 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34703-0x101a91de7b90001 connected
2024-12-06T05:29:37,833 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-06T05:29:37,833 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-06T05:29:37,834 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-06T05:29:37,835 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T05:29:37,835 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34703
2024-12-06T05:29:37,835 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34703
2024-12-06T05:29:37,835 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34703
2024-12-06T05:29:37,836 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34703
2024-12-06T05:29:37,836 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34703
2024-12-06T05:29:37,846 DEBUG [M:0;ec46afd49254:43729 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ec46afd49254:43729
2024-12-06T05:29:37,847 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ec46afd49254,43729,1733462977568
2024-12-06T05:29:37,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:29:37,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:29:37,854 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ec46afd49254,43729,1733462977568
2024-12-06T05:29:37,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-06T05:29:37,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:37,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:37,865 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-06T05:29:37,865 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ec46afd49254,43729,1733462977568 from backup master directory
2024-12-06T05:29:37,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:29:37,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ec46afd49254,43729,1733462977568
2024-12-06T05:29:37,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:29:37,875 WARN  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T05:29:37,875 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ec46afd49254,43729,1733462977568
2024-12-06T05:29:37,882 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/hbase.id] with ID: 8390aa9c-f05a-49bf-ad02-5cb7afbc2c37
2024-12-06T05:29:37,882 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/.tmp/hbase.id
2024-12-06T05:29:37,887 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:37,889 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741826_1002 (size=42)
2024-12-06T05:29:37,889 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741826_1002 (size=42)
2024-12-06T05:29:37,889 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/.tmp/hbase.id]:[hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/hbase.id]
2024-12-06T05:29:37,900 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:29:37,900 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-06T05:29:37,901 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-06T05:29:37,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:37,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:37,915 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741827_1003 (size=196)
2024-12-06T05:29:37,915 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741827_1003 (size=196)
2024-12-06T05:29:38,316 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T05:29:38,317 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-06T05:29:38,317 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:29:38,332 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741828_1004 (size=1189)
2024-12-06T05:29:38,332 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741828_1004 (size=1189)
2024-12-06T05:29:38,713 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:38,735 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store
2024-12-06T05:29:38,746 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741829_1005 (size=34)
2024-12-06T05:29:38,746 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741829_1005 (size=34)
2024-12-06T05:29:38,747 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:29:38,747 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T05:29:38,747 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:29:38,747 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:29:38,747 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T05:29:38,747 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:29:38,747 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:29:38,747 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733462978747Disabling compacts and flushes for region at 1733462978747Disabling writes for close at 1733462978747Writing region close event to WAL at 1733462978747Closed at 1733462978747
2024-12-06T05:29:38,748 WARN  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/.initializing
2024-12-06T05:29:38,748 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/WALs/ec46afd49254,43729,1733462977568
2024-12-06T05:29:38,750 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C43729%2C1733462977568, suffix=, logDir=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/WALs/ec46afd49254,43729,1733462977568, archiveDir=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/oldWALs, maxLogs=10
2024-12-06T05:29:38,750 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C43729%2C1733462977568.1733462978750
2024-12-06T05:29:38,755 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/WALs/ec46afd49254,43729,1733462977568/ec46afd49254%2C43729%2C1733462977568.1733462978750
2024-12-06T05:29:38,755 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35265:35265),(127.0.0.1/127.0.0.1:42051:42051)]
2024-12-06T05:29:38,756 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:29:38,756 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:29:38,756 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:29:38,756 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:29:38,758 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:29:38,759 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-06T05:29:38,759 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:38,760 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:38,760 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:29:38,761 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-06T05:29:38,761 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:38,762 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:29:38,762 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:29:38,763 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-06T05:29:38,763 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:38,763 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:29:38,764 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:29:38,765 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-06T05:29:38,765 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:38,765 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:29:38,766 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:29:38,766 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:29:38,767 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:29:38,769 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:29:38,769 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:29:38,770 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-06T05:29:38,772 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:29:38,776 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:29:38,776 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=719205, jitterRate=-0.08548474311828613}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-06T05:29:38,777 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733462978757Initializing all the Stores at 1733462978757Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462978758 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462978758Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462978758Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462978758Cleaning up temporary data from old regions at 1733462978769 (+11 ms)Region opened successfully at 1733462978777 (+8 ms)
2024-12-06T05:29:38,777 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-06T05:29:38,783 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a4e642e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0
2024-12-06T05:29:38,784 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-06T05:29:38,784 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-06T05:29:38,784 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-06T05:29:38,784 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-06T05:29:38,785 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-06T05:29:38,786 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-06T05:29:38,786 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-06T05:29:38,788 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-06T05:29:38,789 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-06T05:29:38,843 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-06T05:29:38,843 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-06T05:29:38,844 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-06T05:29:38,853 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-06T05:29:38,854 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-06T05:29:38,856 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-06T05:29:38,864 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-06T05:29:38,866 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-06T05:29:38,875 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-06T05:29:38,880 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-06T05:29:38,885 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-06T05:29:38,887 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:38,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T05:29:38,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T05:29:38,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:38,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:38,896 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ec46afd49254,43729,1733462977568, sessionid=0x101a91de7b90000, setting cluster-up flag (Was=false)
2024-12-06T05:29:38,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:38,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:38,949 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-06T05:29:38,951 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec46afd49254,43729,1733462977568
2024-12-06T05:29:38,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:38,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:39,001 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-06T05:29:39,002 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec46afd49254,43729,1733462977568
2024-12-06T05:29:39,004 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-06T05:29:39,005 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-06T05:29:39,005 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-06T05:29:39,005 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-06T05:29:39,006 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ec46afd49254,43729,1733462977568
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-06T05:29:39,007 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:29:39,007 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:29:39,007 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:29:39,007 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:29:39,007 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ec46afd49254:0, corePoolSize=10, maxPoolSize=10
2024-12-06T05:29:39,008 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:39,008 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ec46afd49254:0, corePoolSize=2, maxPoolSize=2
2024-12-06T05:29:39,008 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:39,009 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733463009009
2024-12-06T05:29:39,009 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-06T05:29:39,009 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-06T05:29:39,009 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-06T05:29:39,009 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-06T05:29:39,009 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-06T05:29:39,009 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-06T05:29:39,009 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,010 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:29:39,010 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-06T05:29:39,011 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:39,011 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-06T05:29:39,012 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-06T05:29:39,012 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-06T05:29:39,012 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-06T05:29:39,013 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-06T05:29:39,013 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-06T05:29:39,017 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462979013,5,FailOnTimeoutGroup]
2024-12-06T05:29:39,017 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462979017,5,FailOnTimeoutGroup]
2024-12-06T05:29:39,018 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,018 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-06T05:29:39,018 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,018 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,022 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741831_1007 (size=1321)
2024-12-06T05:29:39,022 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741831_1007 (size=1321)
2024-12-06T05:29:39,023 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-06T05:29:39,023 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040
2024-12-06T05:29:39,029 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741832_1008 (size=32)
2024-12-06T05:29:39,029 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741832_1008 (size=32)
2024-12-06T05:29:39,030 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:29:39,033 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T05:29:39,035 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T05:29:39,035 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:39,035 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:39,035 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T05:29:39,037 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T05:29:39,037 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:39,037 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:39,037 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T05:29:39,039 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.HRegionServer(746): ClusterId : 8390aa9c-f05a-49bf-ad02-5cb7afbc2c37
2024-12-06T05:29:39,039 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T05:29:39,039 DEBUG [RS:0;ec46afd49254:34703 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-06T05:29:39,039 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:39,039 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:39,039 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T05:29:39,041 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T05:29:39,041 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:39,041 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:39,041 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T05:29:39,042 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740
2024-12-06T05:29:39,043 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740
2024-12-06T05:29:39,044 DEBUG [RS:0;ec46afd49254:34703 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-06T05:29:39,044 DEBUG [RS:0;ec46afd49254:34703 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-06T05:29:39,044 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T05:29:39,044 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T05:29:39,045 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T05:29:39,046 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T05:29:39,047 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:29:39,047 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=728741, jitterRate=-0.07335837185382843}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T05:29:39,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733462979030Initializing all the Stores at 1733462979031 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462979031Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462979033 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462979033Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462979033Cleaning up temporary data from old regions at 1733462979044 (+11 ms)Region opened successfully at 1733462979048 (+4 ms)
2024-12-06T05:29:39,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T05:29:39,048 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T05:29:39,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T05:29:39,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T05:29:39,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T05:29:39,048 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T05:29:39,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733462979048Disabling compacts and flushes for region at 1733462979048Disabling writes for close at 1733462979048Writing region close event to WAL at 1733462979048Closed at 1733462979048
2024-12-06T05:29:39,049 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:29:39,050 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-06T05:29:39,050 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-06T05:29:39,051 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T05:29:39,052 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-06T05:29:39,054 DEBUG [RS:0;ec46afd49254:34703 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-06T05:29:39,054 DEBUG [RS:0;ec46afd49254:34703 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b2ce9e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0
2024-12-06T05:29:39,065 DEBUG [RS:0;ec46afd49254:34703 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ec46afd49254:34703
2024-12-06T05:29:39,065 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-06T05:29:39,065 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-06T05:29:39,065 DEBUG [RS:0;ec46afd49254:34703 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-06T05:29:39,066 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec46afd49254,43729,1733462977568 with port=34703, startcode=1733462977823
2024-12-06T05:29:39,066 DEBUG [RS:0;ec46afd49254:34703 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-06T05:29:39,068 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49897, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-12-06T05:29:39,069 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43729 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ec46afd49254,34703,1733462977823
2024-12-06T05:29:39,069 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43729 {}] master.ServerManager(517): Registering regionserver=ec46afd49254,34703,1733462977823
2024-12-06T05:29:39,070 DEBUG [RS:0;ec46afd49254:34703 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040
2024-12-06T05:29:39,070 DEBUG [RS:0;ec46afd49254:34703 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44199
2024-12-06T05:29:39,070 DEBUG [RS:0;ec46afd49254:34703 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-06T05:29:39,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T05:29:39,075 DEBUG [RS:0;ec46afd49254:34703 {}] zookeeper.ZKUtil(111): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec46afd49254,34703,1733462977823
2024-12-06T05:29:39,075 WARN  [RS:0;ec46afd49254:34703 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T05:29:39,075 INFO  [RS:0;ec46afd49254:34703 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:29:39,075 DEBUG [RS:0;ec46afd49254:34703 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/WALs/ec46afd49254,34703,1733462977823
2024-12-06T05:29:39,075 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec46afd49254,34703,1733462977823]
2024-12-06T05:29:39,078 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-06T05:29:39,079 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-06T05:29:39,080 INFO  [RS:0;ec46afd49254:34703 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T05:29:39,080 INFO  [RS:0;ec46afd49254:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,080 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-06T05:29:39,081 INFO  [RS:0;ec46afd49254:34703 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-06T05:29:39,081 INFO  [RS:0;ec46afd49254:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,081 DEBUG [RS:0;ec46afd49254:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:39,081 DEBUG [RS:0;ec46afd49254:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:39,081 DEBUG [RS:0;ec46afd49254:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:39,081 DEBUG [RS:0;ec46afd49254:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:39,081 DEBUG [RS:0;ec46afd49254:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:39,081 DEBUG [RS:0;ec46afd49254:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0, corePoolSize=2, maxPoolSize=2
2024-12-06T05:29:39,081 DEBUG [RS:0;ec46afd49254:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:39,081 DEBUG [RS:0;ec46afd49254:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:39,081 DEBUG [RS:0;ec46afd49254:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:39,081 DEBUG [RS:0;ec46afd49254:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:39,081 DEBUG [RS:0;ec46afd49254:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:39,081 DEBUG [RS:0;ec46afd49254:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:29:39,081 DEBUG [RS:0;ec46afd49254:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3
2024-12-06T05:29:39,081 DEBUG [RS:0;ec46afd49254:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3
2024-12-06T05:29:39,083 INFO  [RS:0;ec46afd49254:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,083 INFO  [RS:0;ec46afd49254:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,083 INFO  [RS:0;ec46afd49254:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,083 INFO  [RS:0;ec46afd49254:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,083 INFO  [RS:0;ec46afd49254:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,083 INFO  [RS:0;ec46afd49254:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,34703,1733462977823-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T05:29:39,101 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-06T05:29:39,101 INFO  [RS:0;ec46afd49254:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,34703,1733462977823-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,102 INFO  [RS:0;ec46afd49254:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,102 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.Replication(171): ec46afd49254,34703,1733462977823 started
2024-12-06T05:29:39,113 INFO  [RS:0;ec46afd49254:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,113 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.HRegionServer(1482): Serving as ec46afd49254,34703,1733462977823, RpcServer on ec46afd49254/172.17.0.3:34703, sessionid=0x101a91de7b90001
2024-12-06T05:29:39,113 DEBUG [RS:0;ec46afd49254:34703 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-06T05:29:39,113 DEBUG [RS:0;ec46afd49254:34703 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec46afd49254,34703,1733462977823
2024-12-06T05:29:39,113 DEBUG [RS:0;ec46afd49254:34703 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,34703,1733462977823'
2024-12-06T05:29:39,113 DEBUG [RS:0;ec46afd49254:34703 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-06T05:29:39,114 DEBUG [RS:0;ec46afd49254:34703 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-06T05:29:39,115 DEBUG [RS:0;ec46afd49254:34703 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-06T05:29:39,115 DEBUG [RS:0;ec46afd49254:34703 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-06T05:29:39,115 DEBUG [RS:0;ec46afd49254:34703 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec46afd49254,34703,1733462977823
2024-12-06T05:29:39,115 DEBUG [RS:0;ec46afd49254:34703 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,34703,1733462977823'
2024-12-06T05:29:39,115 DEBUG [RS:0;ec46afd49254:34703 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-06T05:29:39,115 DEBUG [RS:0;ec46afd49254:34703 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-06T05:29:39,115 DEBUG [RS:0;ec46afd49254:34703 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-06T05:29:39,115 INFO  [RS:0;ec46afd49254:34703 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-06T05:29:39,116 INFO  [RS:0;ec46afd49254:34703 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-06T05:29:39,202 WARN  [ec46afd49254:43729 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-06T05:29:39,218 INFO  [RS:0;ec46afd49254:34703 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C34703%2C1733462977823, suffix=, logDir=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/WALs/ec46afd49254,34703,1733462977823, archiveDir=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/oldWALs, maxLogs=32
2024-12-06T05:29:39,218 INFO  [RS:0;ec46afd49254:34703 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C34703%2C1733462977823.1733462979218
2024-12-06T05:29:39,224 INFO  [RS:0;ec46afd49254:34703 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/WALs/ec46afd49254,34703,1733462977823/ec46afd49254%2C34703%2C1733462977823.1733462979218
2024-12-06T05:29:39,225 DEBUG [RS:0;ec46afd49254:34703 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42051:42051),(127.0.0.1/127.0.0.1:35265:35265)]
2024-12-06T05:29:39,452 DEBUG [ec46afd49254:43729 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-06T05:29:39,453 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ec46afd49254,34703,1733462977823
2024-12-06T05:29:39,455 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec46afd49254,34703,1733462977823, state=OPENING
2024-12-06T05:29:39,506 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-06T05:29:39,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:39,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:29:39,519 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T05:29:39,519 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:29:39,519 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:29:39,519 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ec46afd49254,34703,1733462977823}]
2024-12-06T05:29:39,675 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-06T05:29:39,677 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59973, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-06T05:29:39,681 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-06T05:29:39,682 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:29:39,685 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C34703%2C1733462977823.meta, suffix=.meta, logDir=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/WALs/ec46afd49254,34703,1733462977823, archiveDir=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/oldWALs, maxLogs=32
2024-12-06T05:29:39,686 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C34703%2C1733462977823.meta.1733462979686.meta
2024-12-06T05:29:39,693 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/WALs/ec46afd49254,34703,1733462977823/ec46afd49254%2C34703%2C1733462977823.meta.1733462979686.meta
2024-12-06T05:29:39,697 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42051:42051),(127.0.0.1/127.0.0.1:35265:35265)]
2024-12-06T05:29:39,698 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:29:39,699 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-06T05:29:39,699 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-06T05:29:39,699 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-06T05:29:39,699 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-06T05:29:39,699 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:29:39,699 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-06T05:29:39,699 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-06T05:29:39,701 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T05:29:39,702 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T05:29:39,702 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:39,702 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:39,702 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T05:29:39,703 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T05:29:39,703 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:39,703 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:39,704 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T05:29:39,704 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T05:29:39,704 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:39,705 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:39,705 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T05:29:39,705 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T05:29:39,705 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:39,706 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:29:39,706 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T05:29:39,706 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740
2024-12-06T05:29:39,707 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740
2024-12-06T05:29:39,709 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T05:29:39,709 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T05:29:39,709 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T05:29:39,711 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T05:29:39,712 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=715434, jitterRate=-0.09027968347072601}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T05:29:39,712 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-06T05:29:39,712 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733462979699Writing region info on filesystem at 1733462979699Initializing all the Stores at 1733462979700 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462979700Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462979701 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462979701Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462979701Cleaning up temporary data from old regions at 1733462979709 (+8 ms)Running coprocessor post-open hooks at 1733462979712 (+3 ms)Region opened successfully at 1733462979712
2024-12-06T05:29:39,713 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733462979675
2024-12-06T05:29:39,714 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:39,716 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-06T05:29:39,716 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-06T05:29:39,716 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ec46afd49254,34703,1733462977823
2024-12-06T05:29:39,717 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec46afd49254,34703,1733462977823, state=OPEN
2024-12-06T05:29:39,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T05:29:39,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T05:29:39,760 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ec46afd49254,34703,1733462977823
2024-12-06T05:29:39,760 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:29:39,760 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:29:39,765 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-06T05:29:39,765 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec46afd49254,34703,1733462977823 in 241 msec
2024-12-06T05:29:39,770 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-06T05:29:39,770 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 715 msec
2024-12-06T05:29:39,771 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:29:39,771 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-06T05:29:39,773 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T05:29:39,773 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec46afd49254,34703,1733462977823, seqNum=-1]
2024-12-06T05:29:39,773 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T05:29:39,774 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41275, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T05:29:39,780 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 774 msec
2024-12-06T05:29:39,780 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733462979780, completionTime=-1
2024-12-06T05:29:39,780 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-06T05:29:39,780 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-06T05:29:39,782 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-06T05:29:39,782 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733463039782
2024-12-06T05:29:39,782 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733463099782
2024-12-06T05:29:39,782 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-12-06T05:29:39,782 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43729,1733462977568-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,782 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43729,1733462977568-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,782 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43729,1733462977568-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,782 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ec46afd49254:43729, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,782 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,783 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,784 DEBUG [master/ec46afd49254:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T05:29:39,786 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.911sec
2024-12-06T05:29:39,786 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-06T05:29:39,786 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-06T05:29:39,786 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-06T05:29:39,786 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-06T05:29:39,786 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-06T05:29:39,786 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43729,1733462977568-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T05:29:39,786 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43729,1733462977568-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-06T05:29:39,788 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-06T05:29:39,788 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-06T05:29:39,788 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43729,1733462977568-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:29:39,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e442426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:29:39,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ec46afd49254,43729,-1 for getting cluster id
2024-12-06T05:29:39,840 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-06T05:29:39,844 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8390aa9c-f05a-49bf-ad02-5cb7afbc2c37'
2024-12-06T05:29:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-06T05:29:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8390aa9c-f05a-49bf-ad02-5cb7afbc2c37"

2024-12-06T05:29:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26f1d463, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:29:39,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ec46afd49254,43729,-1]
2024-12-06T05:29:39,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-06T05:29:39,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:29:39,848 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-06T05:29:39,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a63a5a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:29:39,849 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T05:29:39,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec46afd49254,34703,1733462977823, seqNum=-1]
2024-12-06T05:29:39,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T05:29:39,851 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T05:29:39,853 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ec46afd49254,43729,1733462977568
2024-12-06T05:29:39,853 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:29:39,855 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-06T05:29:39,856 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-06T05:29:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is ec46afd49254,43729,1733462977568
2024-12-06T05:29:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@32b03fb8
2024-12-06T05:29:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-06T05:29:39,858 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-06T05:29:39,858 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-06T05:29:39,858 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-06T05:29:39,859 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T05:29:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T05:29:39,861 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-06T05:29:39,862 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:39,862 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-12-06T05:29:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T05:29:39,863 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-06T05:29:39,871 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741835_1011 (size=405)
2024-12-06T05:29:39,871 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741835_1011 (size=405)
2024-12-06T05:29:39,873 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ce9ade53362fadcfc6ea64e084537645, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040
2024-12-06T05:29:39,878 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741836_1012 (size=88)
2024-12-06T05:29:39,879 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741836_1012 (size=88)
2024-12-06T05:29:39,880 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:29:39,880 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing ce9ade53362fadcfc6ea64e084537645, disabling compactions & flushes
2024-12-06T05:29:39,880 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:29:39,880 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:29:39,880 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645. after waiting 0 ms
2024-12-06T05:29:39,880 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:29:39,881 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:29:39,881 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for ce9ade53362fadcfc6ea64e084537645:
Waiting for close lock at 1733462979880Disabling compacts and flushes for region at 1733462979880Disabling writes for close at 1733462979880Writing region close event to WAL at 1733462979880Closed at 1733462979880
2024-12-06T05:29:39,882 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-06T05:29:39,882 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1733462979882"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733462979882"}]},"ts":"1733462979882"}
2024-12-06T05:29:39,884 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-06T05:29:39,886 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-06T05:29:39,886 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733462979886"}]},"ts":"1733462979886"}
2024-12-06T05:29:39,888 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-12-06T05:29:39,888 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:39,888 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=ce9ade53362fadcfc6ea64e084537645, ASSIGN}]
2024-12-06T05:29:39,889 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=ce9ade53362fadcfc6ea64e084537645, ASSIGN
2024-12-06T05:29:39,891 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=ce9ade53362fadcfc6ea64e084537645, ASSIGN; state=OFFLINE, location=ec46afd49254,34703,1733462977823; forceNewPlan=false, retain=false
2024-12-06T05:29:40,042 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ce9ade53362fadcfc6ea64e084537645, regionState=OPENING, regionLocation=ec46afd49254,34703,1733462977823
2024-12-06T05:29:40,047 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=ce9ade53362fadcfc6ea64e084537645, ASSIGN because future has completed
2024-12-06T05:29:40,048 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce9ade53362fadcfc6ea64e084537645, server=ec46afd49254,34703,1733462977823}]
2024-12-06T05:29:40,209 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:29:40,210 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ce9ade53362fadcfc6ea64e084537645, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:29:40,210 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling ce9ade53362fadcfc6ea64e084537645
2024-12-06T05:29:40,210 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:29:40,210 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ce9ade53362fadcfc6ea64e084537645
2024-12-06T05:29:40,210 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ce9ade53362fadcfc6ea64e084537645
2024-12-06T05:29:40,212 INFO  [StoreOpener-ce9ade53362fadcfc6ea64e084537645-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ce9ade53362fadcfc6ea64e084537645 
2024-12-06T05:29:40,214 INFO  [StoreOpener-ce9ade53362fadcfc6ea64e084537645-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce9ade53362fadcfc6ea64e084537645 columnFamilyName info
2024-12-06T05:29:40,215 DEBUG [StoreOpener-ce9ade53362fadcfc6ea64e084537645-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:29:40,215 INFO  [StoreOpener-ce9ade53362fadcfc6ea64e084537645-1 {}] regionserver.HStore(327): Store=ce9ade53362fadcfc6ea64e084537645/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:29:40,216 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ce9ade53362fadcfc6ea64e084537645
2024-12-06T05:29:40,216 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645
2024-12-06T05:29:40,217 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645
2024-12-06T05:29:40,217 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ce9ade53362fadcfc6ea64e084537645
2024-12-06T05:29:40,218 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ce9ade53362fadcfc6ea64e084537645
2024-12-06T05:29:40,220 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ce9ade53362fadcfc6ea64e084537645
2024-12-06T05:29:40,222 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:29:40,223 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ce9ade53362fadcfc6ea64e084537645; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=816630, jitterRate=0.038399383425712585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-06T05:29:40,223 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ce9ade53362fadcfc6ea64e084537645
2024-12-06T05:29:40,223 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ce9ade53362fadcfc6ea64e084537645:
Running coprocessor pre-open hook at 1733462980210Writing region info on filesystem at 1733462980210Initializing all the Stores at 1733462980212 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462980212Cleaning up temporary data from old regions at 1733462980218 (+6 ms)Running coprocessor post-open hooks at 1733462980223 (+5 ms)Region opened successfully at 1733462980223
2024-12-06T05:29:40,225 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645., pid=6, masterSystemTime=1733462980204
2024-12-06T05:29:40,228 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:29:40,228 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:29:40,229 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ce9ade53362fadcfc6ea64e084537645, regionState=OPEN, openSeqNum=2, regionLocation=ec46afd49254,34703,1733462977823
2024-12-06T05:29:40,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce9ade53362fadcfc6ea64e084537645, server=ec46afd49254,34703,1733462977823 because future has completed
2024-12-06T05:29:40,235 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-06T05:29:40,235 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ce9ade53362fadcfc6ea64e084537645, server=ec46afd49254,34703,1733462977823 in 184 msec
2024-12-06T05:29:40,237 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-06T05:29:40,237 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=ce9ade53362fadcfc6ea64e084537645, ASSIGN in 347 msec
2024-12-06T05:29:40,238 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-06T05:29:40,238 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733462980238"}]},"ts":"1733462980238"}
2024-12-06T05:29:40,240 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-12-06T05:29:40,241 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-06T05:29:40,243 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 382 msec
2024-12-06T05:29:40,714 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:40,888 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:41,716 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:41,889 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:42,008 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T05:29:42,010 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:42,010 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:42,011 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:42,011 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:42,012 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:42,012 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:42,033 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:42,033 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:42,033 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:42,033 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:42,033 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:42,034 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:42,036 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:42,036 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:42,036 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:42,038 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:29:42,717 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:42,891 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:43,718 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:43,892 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:44,719 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:44,893 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:45,079 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-06T05:29:45,080 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-12-06T05:29:45,719 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:45,894 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:46,505 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-06T05:29:46,505 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-06T05:29:46,507 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T05:29:46,507 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-06T05:29:46,508 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-06T05:29:46,508 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-06T05:29:46,509 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T05:29:46,509 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-12-06T05:29:46,721 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:46,895 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:47,722 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:47,896 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:48,722 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:48,896 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:49,723 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T05:29:49,868 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-06T05:29:49,868 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-12-06T05:29:49,874 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T05:29:49,874 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:29:49,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645., hostname=ec46afd49254,34703,1733462977823, seqNum=2]
2024-12-06T05:29:49,886 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T05:29:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T05:29:49,892 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-06T05:29:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-12-06T05:29:49,893 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-06T05:29:49,894 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-06T05:29:49,897 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:50,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34703 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-12-06T05:29:50,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:29:50,062 INFO  [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing ce9ade53362fadcfc6ea64e084537645 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-06T05:29:50,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp/info/bf3fa5894b79425d8f614dd5d6a7059c is 1080, key is row0001/info:/1733462989880/Put/seqid=0
2024-12-06T05:29:50,100 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741837_1013 (size=6033)
2024-12-06T05:29:50,101 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741837_1013 (size=6033)
2024-12-06T05:29:50,102 INFO  [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp/info/bf3fa5894b79425d8f614dd5d6a7059c
2024-12-06T05:29:50,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp/info/bf3fa5894b79425d8f614dd5d6a7059c as hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/bf3fa5894b79425d8f614dd5d6a7059c
2024-12-06T05:29:50,119 INFO  [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/bf3fa5894b79425d8f614dd5d6a7059c, entries=1, sequenceid=5, filesize=5.9 K
2024-12-06T05:29:50,120 INFO  [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ce9ade53362fadcfc6ea64e084537645 in 58ms, sequenceid=5, compaction requested=false
2024-12-06T05:29:50,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for ce9ade53362fadcfc6ea64e084537645:

2024-12-06T05:29:50,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:29:50,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-12-06T05:29:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-12-06T05:29:50,128 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-12-06T05:29:50,128 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 231 msec
2024-12-06T05:29:50,130 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 241 msec
2024-12-06T05:29:50,724 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:50,898 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:51,725 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:51,899 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:52,726 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:52,900 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:53,727 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:53,901 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:54,728 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:54,902 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:55,729 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:55,903 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:56,731 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:56,904 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:57,732 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:57,905 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:58,734 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:58,735 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575 after 68068ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:58,906 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:58,908 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta after 68061ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T05:29:59,736 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:59,908 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:29:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-12-06T05:29:59,967 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-06T05:29:59,971 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T05:29:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T05:29:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-06T05:29:59,974 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-06T05:29:59,976 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-06T05:29:59,976 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-06T05:30:00,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34703 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-12-06T05:30:00,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:30:00,132 INFO  [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing ce9ade53362fadcfc6ea64e084537645 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-06T05:30:00,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp/info/a18e46dc5f914fecb3c72a57c4009848 is 1080, key is row0002/info:/1733462999968/Put/seqid=0
2024-12-06T05:30:00,155 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741838_1014 (size=6033)
2024-12-06T05:30:00,155 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741838_1014 (size=6033)
2024-12-06T05:30:00,155 INFO  [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp/info/a18e46dc5f914fecb3c72a57c4009848
2024-12-06T05:30:00,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp/info/a18e46dc5f914fecb3c72a57c4009848 as hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/a18e46dc5f914fecb3c72a57c4009848
2024-12-06T05:30:00,170 INFO  [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/a18e46dc5f914fecb3c72a57c4009848, entries=1, sequenceid=9, filesize=5.9 K
2024-12-06T05:30:00,171 INFO  [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ce9ade53362fadcfc6ea64e084537645 in 39ms, sequenceid=9, compaction requested=false
2024-12-06T05:30:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for ce9ade53362fadcfc6ea64e084537645:

2024-12-06T05:30:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:30:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-12-06T05:30:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-12-06T05:30:00,176 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-12-06T05:30:00,176 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 197 msec
2024-12-06T05:30:00,179 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 206 msec
2024-12-06T05:30:00,736 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:00,909 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:01,737 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:01,910 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:02,738 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:02,910 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:03,739 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:03,911 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:04,739 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:04,912 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:05,740 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:05,912 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:06,741 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:06,913 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:07,547 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T05:30:07,741 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:07,914 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:08,742 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:08,914 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:09,742 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:09,915 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-06T05:30:10,027 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-06T05:30:10,030 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C34703%2C1733462977823.1733463010029
2024-12-06T05:30:10,035 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:10,035 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:10,036 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:10,036 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:10,036 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:10,036 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/WALs/ec46afd49254,34703,1733462977823/ec46afd49254%2C34703%2C1733462977823.1733462979218 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/WALs/ec46afd49254,34703,1733462977823/ec46afd49254%2C34703%2C1733462977823.1733463010029
2024-12-06T05:30:10,037 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42051:42051),(127.0.0.1/127.0.0.1:35265:35265)]
2024-12-06T05:30:10,037 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/WALs/ec46afd49254,34703,1733462977823/ec46afd49254%2C34703%2C1733462977823.1733462979218 is not closed yet, will try archiving it next time
2024-12-06T05:30:10,038 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T05:30:10,039 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741833_1009 (size=5546)
2024-12-06T05:30:10,039 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741833_1009 (size=5546)
2024-12-06T05:30:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T05:30:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-06T05:30:10,041 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-06T05:30:10,043 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-06T05:30:10,043 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-06T05:30:10,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34703 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-12-06T05:30:10,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:30:10,198 INFO  [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing ce9ade53362fadcfc6ea64e084537645 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-06T05:30:10,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp/info/59f3309b46784a45a7bd48814dcba05a is 1080, key is row0003/info:/1733463010028/Put/seqid=0
2024-12-06T05:30:10,214 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741840_1016 (size=6033)
2024-12-06T05:30:10,214 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741840_1016 (size=6033)
2024-12-06T05:30:10,214 INFO  [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp/info/59f3309b46784a45a7bd48814dcba05a
2024-12-06T05:30:10,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp/info/59f3309b46784a45a7bd48814dcba05a as hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/59f3309b46784a45a7bd48814dcba05a
2024-12-06T05:30:10,227 INFO  [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/59f3309b46784a45a7bd48814dcba05a, entries=1, sequenceid=13, filesize=5.9 K
2024-12-06T05:30:10,228 INFO  [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ce9ade53362fadcfc6ea64e084537645 in 31ms, sequenceid=13, compaction requested=true
2024-12-06T05:30:10,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for ce9ade53362fadcfc6ea64e084537645:

2024-12-06T05:30:10,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:30:10,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-12-06T05:30:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-12-06T05:30:10,258 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-12-06T05:30:10,258 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 210 msec
2024-12-06T05:30:10,262 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 221 msec
2024-12-06T05:30:10,743 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:10,916 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:11,744 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:11,917 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:12,745 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:12,917 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:13,746 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:13,918 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:14,747 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:14,919 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:15,748 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:15,920 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:16,748 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:16,920 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:17,749 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:17,921 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:18,750 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:18,922 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:19,751 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:19,806 INFO  [master/ec46afd49254:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-06T05:30:19,806 INFO  [master/ec46afd49254:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-06T05:30:19,922 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-06T05:30:20,117 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-06T05:30:20,117 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T05:30:20,119 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T05:30:20,120 DEBUG [Time-limited test {}] regionserver.HStore(1541): ce9ade53362fadcfc6ea64e084537645/info is initiating minor compaction (all files)
2024-12-06T05:30:20,120 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T05:30:20,120 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:20,120 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of ce9ade53362fadcfc6ea64e084537645/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:30:20,120 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/bf3fa5894b79425d8f614dd5d6a7059c, hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/a18e46dc5f914fecb3c72a57c4009848, hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/59f3309b46784a45a7bd48814dcba05a] into tmpdir=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp, totalSize=17.7 K
2024-12-06T05:30:20,121 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting bf3fa5894b79425d8f614dd5d6a7059c, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733462989880
2024-12-06T05:30:20,121 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a18e46dc5f914fecb3c72a57c4009848, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1733462999968
2024-12-06T05:30:20,122 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 59f3309b46784a45a7bd48814dcba05a, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733463010028
2024-12-06T05:30:20,137 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): ce9ade53362fadcfc6ea64e084537645#info#compaction#45 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T05:30:20,138 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp/info/ee6e38f5db134be3addbdf401327a04e is 1080, key is row0001/info:/1733462989880/Put/seqid=0
2024-12-06T05:30:20,144 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741841_1017 (size=8296)
2024-12-06T05:30:20,144 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741841_1017 (size=8296)
2024-12-06T05:30:20,150 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp/info/ee6e38f5db134be3addbdf401327a04e as hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/ee6e38f5db134be3addbdf401327a04e
2024-12-06T05:30:20,157 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce9ade53362fadcfc6ea64e084537645/info of ce9ade53362fadcfc6ea64e084537645 into ee6e38f5db134be3addbdf401327a04e(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T05:30:20,157 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for ce9ade53362fadcfc6ea64e084537645:

2024-12-06T05:30:20,160 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C34703%2C1733462977823.1733463020160
2024-12-06T05:30:20,169 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:20,169 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:20,169 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:20,169 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:20,170 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:20,170 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/WALs/ec46afd49254,34703,1733462977823/ec46afd49254%2C34703%2C1733462977823.1733463010029 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/WALs/ec46afd49254,34703,1733462977823/ec46afd49254%2C34703%2C1733462977823.1733463020160
2024-12-06T05:30:20,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741839_1015 (size=2520)
2024-12-06T05:30:20,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741839_1015 (size=2520)
2024-12-06T05:30:20,176 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42051:42051),(127.0.0.1/127.0.0.1:35265:35265)]
2024-12-06T05:30:20,176 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/WALs/ec46afd49254,34703,1733462977823/ec46afd49254%2C34703%2C1733462977823.1733462979218 to hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/oldWALs/ec46afd49254%2C34703%2C1733462977823.1733462979218
2024-12-06T05:30:20,177 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T05:30:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T05:30:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-06T05:30:20,179 INFO  [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-06T05:30:20,180 INFO  [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-06T05:30:20,180 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-06T05:30:20,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34703 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-12-06T05:30:20,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:30:20,334 INFO  [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing ce9ade53362fadcfc6ea64e084537645 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-06T05:30:20,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp/info/1672938976f646e59fbd6f11c8565614 is 1080, key is row0000/info:/1733463020158/Put/seqid=0
2024-12-06T05:30:20,345 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741843_1019 (size=6033)
2024-12-06T05:30:20,345 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741843_1019 (size=6033)
2024-12-06T05:30:20,348 INFO  [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp/info/1672938976f646e59fbd6f11c8565614
2024-12-06T05:30:20,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp/info/1672938976f646e59fbd6f11c8565614 as hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/1672938976f646e59fbd6f11c8565614
2024-12-06T05:30:20,361 INFO  [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/1672938976f646e59fbd6f11c8565614, entries=1, sequenceid=18, filesize=5.9 K
2024-12-06T05:30:20,362 INFO  [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ce9ade53362fadcfc6ea64e084537645 in 28ms, sequenceid=18, compaction requested=false
2024-12-06T05:30:20,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for ce9ade53362fadcfc6ea64e084537645:

2024-12-06T05:30:20,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:30:20,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-12-06T05:30:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-12-06T05:30:20,367 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-12-06T05:30:20,367 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 184 msec
2024-12-06T05:30:20,369 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 190 msec
2024-12-06T05:30:20,751 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:20,923 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:21,752 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:21,924 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:22,753 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:22,924 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:23,754 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:23,925 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:24,754 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:24,926 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:25,210 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region ce9ade53362fadcfc6ea64e084537645, had cached 0 bytes from a total of 14329
2024-12-06T05:30:25,755 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:25,927 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:26,756 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:26,927 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:27,756 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:27,928 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:28,757 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:28,929 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:29,758 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:29,930 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:30,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43729 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-06T05:30:30,186 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-06T05:30:30,189 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C34703%2C1733462977823.1733463030189
2024-12-06T05:30:30,196 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,196 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,196 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,196 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,196 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,196 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/WALs/ec46afd49254,34703,1733462977823/ec46afd49254%2C34703%2C1733462977823.1733463020160 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/WALs/ec46afd49254,34703,1733462977823/ec46afd49254%2C34703%2C1733462977823.1733463030189
2024-12-06T05:30:30,197 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42051:42051),(127.0.0.1/127.0.0.1:35265:35265)]
2024-12-06T05:30:30,197 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/WALs/ec46afd49254,34703,1733462977823/ec46afd49254%2C34703%2C1733462977823.1733463020160 is not closed yet, will try archiving it next time
2024-12-06T05:30:30,197 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/WALs/ec46afd49254,34703,1733462977823/ec46afd49254%2C34703%2C1733462977823.1733463010029 to hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/oldWALs/ec46afd49254%2C34703%2C1733462977823.1733463010029
2024-12-06T05:30:30,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-06T05:30:30,197 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T05:30:30,198 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:30:30,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:30:30,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:30:30,198 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-06T05:30:30,198 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-06T05:30:30,198 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=415953718, stopped=false
2024-12-06T05:30:30,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741842_1018 (size=2026)
2024-12-06T05:30:30,198 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ec46afd49254,43729,1733462977568
2024-12-06T05:30:30,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741842_1018 (size=2026)
2024-12-06T05:30:30,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T05:30:30,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T05:30:30,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:30:30,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:30:30,295 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T05:30:30,295 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T05:30:30,296 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:30:30,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:30:30,296 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:30:30,296 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:30:30,296 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ec46afd49254,34703,1733462977823' *****
2024-12-06T05:30:30,296 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-06T05:30:30,297 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-06T05:30:30,297 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-06T05:30:30,297 INFO  [RS:0;ec46afd49254:34703 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-06T05:30:30,297 INFO  [RS:0;ec46afd49254:34703 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-06T05:30:30,297 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.HRegionServer(3091): Received CLOSE for ce9ade53362fadcfc6ea64e084537645
2024-12-06T05:30:30,297 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.HRegionServer(959): stopping server ec46afd49254,34703,1733462977823
2024-12-06T05:30:30,298 INFO  [RS:0;ec46afd49254:34703 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T05:30:30,298 INFO  [RS:0;ec46afd49254:34703 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ec46afd49254:34703.
2024-12-06T05:30:30,298 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ce9ade53362fadcfc6ea64e084537645, disabling compactions & flushes
2024-12-06T05:30:30,298 DEBUG [RS:0;ec46afd49254:34703 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:30:30,298 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:30:30,298 DEBUG [RS:0;ec46afd49254:34703 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:30:30,298 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:30:30,298 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645. after waiting 0 ms
2024-12-06T05:30:30,298 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:30:30,298 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-06T05:30:30,298 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-06T05:30:30,298 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-06T05:30:30,298 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ce9ade53362fadcfc6ea64e084537645 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-06T05:30:30,298 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-06T05:30:30,299 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-06T05:30:30,299 DEBUG [RS:0;ec46afd49254:34703 {}] regionserver.HRegionServer(1325): Online Regions={ce9ade53362fadcfc6ea64e084537645=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645., 1588230740=hbase:meta,,1.1588230740}
2024-12-06T05:30:30,299 DEBUG [RS:0;ec46afd49254:34703 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ce9ade53362fadcfc6ea64e084537645
2024-12-06T05:30:30,299 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T05:30:30,299 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T05:30:30,299 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T05:30:30,299 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T05:30:30,299 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T05:30:30,299 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-12-06T05:30:30,303 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp/info/b5fd736a2ae04c438f98a87bd583f975 is 1080, key is row0001/info:/1733463030187/Put/seqid=0
2024-12-06T05:30:30,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741845_1021 (size=6033)
2024-12-06T05:30:30,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741845_1021 (size=6033)
2024-12-06T05:30:30,309 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp/info/b5fd736a2ae04c438f98a87bd583f975
2024-12-06T05:30:30,316 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/.tmp/info/b5fd736a2ae04c438f98a87bd583f975 as hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/b5fd736a2ae04c438f98a87bd583f975
2024-12-06T05:30:30,318 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740/.tmp/info/bc14c7ebd468422196431686166603d7 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645./info:regioninfo/1733462980228/Put/seqid=0
2024-12-06T05:30:30,322 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/b5fd736a2ae04c438f98a87bd583f975, entries=1, sequenceid=22, filesize=5.9 K
2024-12-06T05:30:30,323 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ce9ade53362fadcfc6ea64e084537645 in 25ms, sequenceid=22, compaction requested=true
2024-12-06T05:30:30,324 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/bf3fa5894b79425d8f614dd5d6a7059c, hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/a18e46dc5f914fecb3c72a57c4009848, hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/59f3309b46784a45a7bd48814dcba05a] to archive
2024-12-06T05:30:30,325 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-06T05:30:30,326 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741846_1022 (size=7308)
2024-12-06T05:30:30,326 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741846_1022 (size=7308)
2024-12-06T05:30:30,327 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740/.tmp/info/bc14c7ebd468422196431686166603d7
2024-12-06T05:30:30,327 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/bf3fa5894b79425d8f614dd5d6a7059c to hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/bf3fa5894b79425d8f614dd5d6a7059c
2024-12-06T05:30:30,328 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/a18e46dc5f914fecb3c72a57c4009848 to hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/a18e46dc5f914fecb3c72a57c4009848
2024-12-06T05:30:30,329 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/59f3309b46784a45a7bd48814dcba05a to hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/info/59f3309b46784a45a7bd48814dcba05a
2024-12-06T05:30:30,330 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ec46afd49254:43729 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-06T05:30:30,330 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [bf3fa5894b79425d8f614dd5d6a7059c=6033, a18e46dc5f914fecb3c72a57c4009848=6033, 59f3309b46784a45a7bd48814dcba05a=6033]
2024-12-06T05:30:30,337 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ce9ade53362fadcfc6ea64e084537645/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-12-06T05:30:30,337 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:30:30,337 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ce9ade53362fadcfc6ea64e084537645:
Waiting for close lock at 1733463030298Running coprocessor pre-close hooks at 1733463030298Disabling compacts and flushes for region at 1733463030298Disabling writes for close at 1733463030298Obtaining lock to block concurrent updates at 1733463030298Preparing flush snapshotting stores in ce9ade53362fadcfc6ea64e084537645 at 1733463030298Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1733463030298Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645. at 1733463030299 (+1 ms)Flushing ce9ade53362fadcfc6ea64e084537645/info: creating writer at 1733463030299Flushing ce9ade53362fadcfc6ea64e084537645/info: appending metadata at 1733463030303 (+4 ms)Flushing ce9ade53362fadcfc6ea64e084537645/info: closing flushed file at 1733463030303Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cbbf0ee: reopening flushed file at 1733463030315 (+12 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ce9ade53362fadcfc6ea64e084537645 in 25ms, sequenceid=22, compaction requested=true at 1733463030323 (+8 ms)Writing region close event to WAL at 1733463030331 (+8 ms)Running coprocessor post-close hooks at 1733463030337 (+6 ms)Closed at 1733463030337
2024-12-06T05:30:30,337 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733462979858.ce9ade53362fadcfc6ea64e084537645.
2024-12-06T05:30:30,350 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740/.tmp/ns/ca022f147aa4458cb937efad6128e8ed is 43, key is default/ns:d/1733462979775/Put/seqid=0
2024-12-06T05:30:30,354 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741847_1023 (size=5153)
2024-12-06T05:30:30,355 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741847_1023 (size=5153)
2024-12-06T05:30:30,355 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740/.tmp/ns/ca022f147aa4458cb937efad6128e8ed
2024-12-06T05:30:30,376 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740/.tmp/table/3cef03d1536c4689ad01e1e47449dba6 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1733462980238/Put/seqid=0
2024-12-06T05:30:30,380 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741848_1024 (size=5508)
2024-12-06T05:30:30,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741848_1024 (size=5508)
2024-12-06T05:30:30,381 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740/.tmp/table/3cef03d1536c4689ad01e1e47449dba6
2024-12-06T05:30:30,388 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740/.tmp/info/bc14c7ebd468422196431686166603d7 as hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740/info/bc14c7ebd468422196431686166603d7
2024-12-06T05:30:30,395 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740/info/bc14c7ebd468422196431686166603d7, entries=10, sequenceid=11, filesize=7.1 K
2024-12-06T05:30:30,396 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740/.tmp/ns/ca022f147aa4458cb937efad6128e8ed as hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740/ns/ca022f147aa4458cb937efad6128e8ed
2024-12-06T05:30:30,401 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740/ns/ca022f147aa4458cb937efad6128e8ed, entries=2, sequenceid=11, filesize=5.0 K
2024-12-06T05:30:30,402 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740/.tmp/table/3cef03d1536c4689ad01e1e47449dba6 as hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740/table/3cef03d1536c4689ad01e1e47449dba6
2024-12-06T05:30:30,408 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740/table/3cef03d1536c4689ad01e1e47449dba6, entries=2, sequenceid=11, filesize=5.4 K
2024-12-06T05:30:30,409 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 110ms, sequenceid=11, compaction requested=false
2024-12-06T05:30:30,420 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-06T05:30:30,420 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T05:30:30,420 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T05:30:30,421 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733463030299Running coprocessor pre-close hooks at 1733463030299Disabling compacts and flushes for region at 1733463030299Disabling writes for close at 1733463030299Obtaining lock to block concurrent updates at 1733463030299Preparing flush snapshotting stores in 1588230740 at 1733463030299Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1733463030300 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733463030301 (+1 ms)Flushing 1588230740/info: creating writer at 1733463030301Flushing 1588230740/info: appending metadata at 1733463030318 (+17 ms)Flushing 1588230740/info: closing flushed file at 1733463030318Flushing 1588230740/ns: creating writer at 1733463030332 (+14 ms)Flushing 1588230740/ns: appending metadata at 1733463030349 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1733463030349Flushing 1588230740/table: creating writer at 1733463030360 (+11 ms)Flushing 1588230740/table: appending metadata at 1733463030375 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733463030375Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d72a7bc: reopening flushed file at 1733463030387 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41b8e095: reopening flushed file at 1733463030395 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b0cc478: reopening flushed file at 1733463030401 (+6 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 110ms, sequenceid=11, compaction requested=false at 1733463030409 (+8 ms)Writing region close event to WAL at 1733463030416 (+7 ms)Running coprocessor post-close hooks at 1733463030420 (+4 ms)Closed at 1733463030420
2024-12-06T05:30:30,421 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-06T05:30:30,499 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.HRegionServer(976): stopping server ec46afd49254,34703,1733462977823; all regions closed.
2024-12-06T05:30:30,499 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,499 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,500 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,500 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,500 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,502 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741834_1010 (size=3306)
2024-12-06T05:30:30,503 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741834_1010 (size=3306)
2024-12-06T05:30:30,506 DEBUG [RS:0;ec46afd49254:34703 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/oldWALs
2024-12-06T05:30:30,506 INFO  [RS:0;ec46afd49254:34703 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ec46afd49254%2C34703%2C1733462977823.meta:.meta(num 1733462979686)
2024-12-06T05:30:30,506 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,507 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,507 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,507 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,507 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,509 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741844_1020 (size=1252)
2024-12-06T05:30:30,509 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741844_1020 (size=1252)
2024-12-06T05:30:30,513 DEBUG [RS:0;ec46afd49254:34703 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/oldWALs
2024-12-06T05:30:30,513 INFO  [RS:0;ec46afd49254:34703 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ec46afd49254%2C34703%2C1733462977823:(num 1733463030189)
2024-12-06T05:30:30,513 DEBUG [RS:0;ec46afd49254:34703 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:30:30,513 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T05:30:30,513 INFO  [RS:0;ec46afd49254:34703 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T05:30:30,513 INFO  [RS:0;ec46afd49254:34703 {}] hbase.ChoreService(370): Chore service for: regionserver/ec46afd49254:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-06T05:30:30,514 INFO  [RS:0;ec46afd49254:34703 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T05:30:30,514 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T05:30:30,514 INFO  [RS:0;ec46afd49254:34703 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34703
2024-12-06T05:30:30,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec46afd49254,34703,1733462977823
2024-12-06T05:30:30,534 INFO  [RS:0;ec46afd49254:34703 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T05:30:30,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T05:30:30,534 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher.
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$364/0x00007fc700903c30@32a507bc rejected from java.util.concurrent.ThreadPoolExecutor@77a78f8c[Shutting down, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 14]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?]
	at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4]
2024-12-06T05:30:30,535 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec46afd49254,34703,1733462977823]
2024-12-06T05:30:30,555 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ec46afd49254,34703,1733462977823 already deleted, retry=false
2024-12-06T05:30:30,555 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ec46afd49254,34703,1733462977823 expired; onlineServers=0
2024-12-06T05:30:30,555 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ec46afd49254,43729,1733462977568' *****
2024-12-06T05:30:30,555 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-06T05:30:30,555 INFO  [M:0;ec46afd49254:43729 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T05:30:30,555 INFO  [M:0;ec46afd49254:43729 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T05:30:30,556 DEBUG [M:0;ec46afd49254:43729 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-06T05:30:30,556 DEBUG [M:0;ec46afd49254:43729 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-06T05:30:30,556 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-06T05:30:30,556 DEBUG [master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462979013 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462979013,5,FailOnTimeoutGroup]
2024-12-06T05:30:30,556 DEBUG [master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462979017 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462979017,5,FailOnTimeoutGroup]
2024-12-06T05:30:30,556 INFO  [M:0;ec46afd49254:43729 {}] hbase.ChoreService(370): Chore service for: master/ec46afd49254:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-06T05:30:30,556 INFO  [M:0;ec46afd49254:43729 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T05:30:30,556 DEBUG [M:0;ec46afd49254:43729 {}] master.HMaster(1795): Stopping service threads
2024-12-06T05:30:30,556 INFO  [M:0;ec46afd49254:43729 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-06T05:30:30,556 INFO  [M:0;ec46afd49254:43729 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T05:30:30,556 INFO  [M:0;ec46afd49254:43729 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-06T05:30:30,556 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-06T05:30:30,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-06T05:30:30,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:30:30,566 DEBUG [M:0;ec46afd49254:43729 {}] zookeeper.ZKUtil(347): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-06T05:30:30,566 WARN  [M:0;ec46afd49254:43729 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-06T05:30:30,567 INFO  [M:0;ec46afd49254:43729 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/.lastflushedseqids
2024-12-06T05:30:30,571 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741849_1025 (size=130)
2024-12-06T05:30:30,572 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741849_1025 (size=130)
2024-12-06T05:30:30,572 INFO  [M:0;ec46afd49254:43729 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-06T05:30:30,572 INFO  [M:0;ec46afd49254:43729 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-06T05:30:30,572 DEBUG [M:0;ec46afd49254:43729 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T05:30:30,572 INFO  [M:0;ec46afd49254:43729 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:30:30,572 DEBUG [M:0;ec46afd49254:43729 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:30:30,572 DEBUG [M:0;ec46afd49254:43729 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T05:30:30,572 DEBUG [M:0;ec46afd49254:43729 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:30:30,572 INFO  [M:0;ec46afd49254:43729 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.57 KB heapSize=54.96 KB
2024-12-06T05:30:30,590 DEBUG [M:0;ec46afd49254:43729 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a96c48673be7499a8a38d6e69e7bca50 is 82, key is hbase:meta,,1/info:regioninfo/1733462979716/Put/seqid=0
2024-12-06T05:30:30,594 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741850_1026 (size=5672)
2024-12-06T05:30:30,595 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741850_1026 (size=5672)
2024-12-06T05:30:30,595 INFO  [M:0;ec46afd49254:43729 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a96c48673be7499a8a38d6e69e7bca50
2024-12-06T05:30:30,616 DEBUG [M:0;ec46afd49254:43729 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1505a17f9ed14c58a86532f17d00937d is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733462980242/Put/seqid=0
2024-12-06T05:30:30,623 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741851_1027 (size=7820)
2024-12-06T05:30:30,623 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741851_1027 (size=7820)
2024-12-06T05:30:30,623 INFO  [M:0;ec46afd49254:43729 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.96 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1505a17f9ed14c58a86532f17d00937d
2024-12-06T05:30:30,628 INFO  [M:0;ec46afd49254:43729 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1505a17f9ed14c58a86532f17d00937d
2024-12-06T05:30:30,644 DEBUG [M:0;ec46afd49254:43729 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/10daac0e3b294aaf87deeb508e95b3bf is 69, key is ec46afd49254,34703,1733462977823/rs:state/1733462979069/Put/seqid=0
2024-12-06T05:30:30,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:30:30,645 INFO  [RS:0;ec46afd49254:34703 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T05:30:30,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x101a91de7b90001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:30:30,645 INFO  [RS:0;ec46afd49254:34703 {}] regionserver.HRegionServer(1031): Exiting; stopping=ec46afd49254,34703,1733462977823; zookeeper connection closed.
2024-12-06T05:30:30,645 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@69f2a651 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@69f2a651
2024-12-06T05:30:30,646 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-06T05:30:30,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741852_1028 (size=5156)
2024-12-06T05:30:30,650 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741852_1028 (size=5156)
2024-12-06T05:30:30,650 INFO  [M:0;ec46afd49254:43729 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/10daac0e3b294aaf87deeb508e95b3bf
2024-12-06T05:30:30,675 DEBUG [M:0;ec46afd49254:43729 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/346453621cae4512aabc9effa9c52e2d is 52, key is load_balancer_on/state:d/1733462979854/Put/seqid=0
2024-12-06T05:30:30,679 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741853_1029 (size=5056)
2024-12-06T05:30:30,679 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741853_1029 (size=5056)
2024-12-06T05:30:30,680 INFO  [M:0;ec46afd49254:43729 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/346453621cae4512aabc9effa9c52e2d
2024-12-06T05:30:30,685 DEBUG [M:0;ec46afd49254:43729 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a96c48673be7499a8a38d6e69e7bca50 as hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a96c48673be7499a8a38d6e69e7bca50
2024-12-06T05:30:30,692 INFO  [M:0;ec46afd49254:43729 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a96c48673be7499a8a38d6e69e7bca50, entries=8, sequenceid=121, filesize=5.5 K
2024-12-06T05:30:30,693 DEBUG [M:0;ec46afd49254:43729 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1505a17f9ed14c58a86532f17d00937d as hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1505a17f9ed14c58a86532f17d00937d
2024-12-06T05:30:30,698 INFO  [M:0;ec46afd49254:43729 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1505a17f9ed14c58a86532f17d00937d
2024-12-06T05:30:30,698 INFO  [M:0;ec46afd49254:43729 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1505a17f9ed14c58a86532f17d00937d, entries=14, sequenceid=121, filesize=7.6 K
2024-12-06T05:30:30,699 DEBUG [M:0;ec46afd49254:43729 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/10daac0e3b294aaf87deeb508e95b3bf as hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/10daac0e3b294aaf87deeb508e95b3bf
2024-12-06T05:30:30,704 INFO  [M:0;ec46afd49254:43729 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/10daac0e3b294aaf87deeb508e95b3bf, entries=1, sequenceid=121, filesize=5.0 K
2024-12-06T05:30:30,706 DEBUG [M:0;ec46afd49254:43729 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/346453621cae4512aabc9effa9c52e2d as hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/346453621cae4512aabc9effa9c52e2d
2024-12-06T05:30:30,712 INFO  [M:0;ec46afd49254:43729 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44199/user/jenkins/test-data/1532f034-2b51-8d19-4851-163ac7b12040/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/346453621cae4512aabc9effa9c52e2d, entries=1, sequenceid=121, filesize=4.9 K
2024-12-06T05:30:30,713 INFO  [M:0;ec46afd49254:43729 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.57 KB/44611, heapSize ~54.90 KB/56216, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 141ms, sequenceid=121, compaction requested=false
2024-12-06T05:30:30,717 INFO  [M:0;ec46afd49254:43729 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:30:30,717 DEBUG [M:0;ec46afd49254:43729 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733463030572Disabling compacts and flushes for region at 1733463030572Disabling writes for close at 1733463030572Obtaining lock to block concurrent updates at 1733463030572Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733463030572Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44611, getHeapSize=56216, getOffHeapSize=0, getCellsCount=140 at 1733463030573 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733463030573Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733463030573Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733463030589 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733463030589Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733463030600 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733463030615 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733463030615Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733463030629 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733463030644 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733463030644Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733463030656 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733463030674 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733463030674Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@356d80e5: reopening flushed file at 1733463030684 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69ef3286: reopening flushed file at 1733463030692 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1433a36f: reopening flushed file at 1733463030698 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d78a263: reopening flushed file at 1733463030705 (+7 ms)Finished flush of dataSize ~43.57 KB/44611, heapSize ~54.90 KB/56216, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 141ms, sequenceid=121, compaction requested=false at 1733463030713 (+8 ms)Writing region close event to WAL at 1733463030717 (+4 ms)Closed at 1733463030717
2024-12-06T05:30:30,718 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,718 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,718 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,718 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,718 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:30:30,720 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46681 is added to blk_1073741830_1006 (size=53008)
2024-12-06T05:30:30,720 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35385 is added to blk_1073741830_1006 (size=53008)
2024-12-06T05:30:30,721 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T05:30:30,721 INFO  [M:0;ec46afd49254:43729 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-06T05:30:30,721 INFO  [M:0;ec46afd49254:43729 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43729
2024-12-06T05:30:30,721 INFO  [M:0;ec46afd49254:43729 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T05:30:30,758 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:30,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:30:30,834 INFO  [M:0;ec46afd49254:43729 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T05:30:30,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43729-0x101a91de7b90000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:30:30,836 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4e363d0c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:30:30,836 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5daea1f2{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:30:30,836 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:30:30,836 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6623a859{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:30:30,837 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3dca8de9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/hadoop.log.dir/,STOPPED}
2024-12-06T05:30:30,838 WARN  [BP-1266242495-172.17.0.3-1733462975061 heartbeating to localhost/127.0.0.1:44199 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:30:30,838 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:30:30,838 WARN  [BP-1266242495-172.17.0.3-1733462975061 heartbeating to localhost/127.0.0.1:44199 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1266242495-172.17.0.3-1733462975061 (Datanode Uuid d5890785-ece2-4fa7-b8b3-c610ba90c834) service to localhost/127.0.0.1:44199
2024-12-06T05:30:30,838 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:30:30,838 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/cluster_b8ebfc14-66ba-4c28-dc87-1aa08dd53a02/data/data3/current/BP-1266242495-172.17.0.3-1733462975061 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:30:30,839 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/cluster_b8ebfc14-66ba-4c28-dc87-1aa08dd53a02/data/data4/current/BP-1266242495-172.17.0.3-1733462975061 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:30:30,839 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:30:30,841 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@10fa3dab{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:30:30,841 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@bc64a6a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:30:30,841 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:30:30,841 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4911f4c5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:30:30,841 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69ff6367{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/hadoop.log.dir/,STOPPED}
2024-12-06T05:30:30,842 WARN  [BP-1266242495-172.17.0.3-1733462975061 heartbeating to localhost/127.0.0.1:44199 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:30:30,842 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:30:30,842 WARN  [BP-1266242495-172.17.0.3-1733462975061 heartbeating to localhost/127.0.0.1:44199 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1266242495-172.17.0.3-1733462975061 (Datanode Uuid a8a1f2d0-1b84-418d-8220-695b757af27f) service to localhost/127.0.0.1:44199
2024-12-06T05:30:30,842 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:30:30,843 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/cluster_b8ebfc14-66ba-4c28-dc87-1aa08dd53a02/data/data1/current/BP-1266242495-172.17.0.3-1733462975061 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:30:30,843 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/cluster_b8ebfc14-66ba-4c28-dc87-1aa08dd53a02/data/data2/current/BP-1266242495-172.17.0.3-1733462975061 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:30:30,843 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:30:30,850 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1b5fa12b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T05:30:30,851 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@10128232{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:30:30,851 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:30:30,851 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4cf5e3df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:30:30,851 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46a495b0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/hadoop.log.dir/,STOPPED}
2024-12-06T05:30:30,856 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-06T05:30:30,875 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-06T05:30:30,885 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=206 (was 180)
Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44199 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44199 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:44199
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44199
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44199
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44199
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:44199
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/ec46afd49254:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44199 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=485 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=254 (was 230) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6455 (was 7095)
2024-12-06T05:30:30,894 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=205, OpenFileDescriptor=485, MaxFileDescriptor=1048576, SystemLoadAverage=254, ProcessCount=11, AvailableMemoryMB=6454
2024-12-06T05:30:30,894 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-06T05:30:30,894 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/hadoop.log.dir so I do NOT create it in target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe
2024-12-06T05:30:30,894 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3d59f77f-3cda-cec0-8179-b55bb5c6c536/hadoop.tmp.dir so I do NOT create it in target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe
2024-12-06T05:30:30,894 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/cluster_aa9f0358-d841-213d-3fe9-e6bb995c275f, deleteOnExit=true
2024-12-06T05:30:30,894 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-06T05:30:30,894 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/test.cache.data in system properties and HBase conf
2024-12-06T05:30:30,895 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/hadoop.tmp.dir in system properties and HBase conf
2024-12-06T05:30:30,895 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/hadoop.log.dir in system properties and HBase conf
2024-12-06T05:30:30,895 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-06T05:30:30,895 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-06T05:30:30,895 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-06T05:30:30,895 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-06T05:30:30,895 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-06T05:30:30,895 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-06T05:30:30,895 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-06T05:30:30,895 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T05:30:30,895 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-06T05:30:30,896 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-06T05:30:30,896 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T05:30:30,896 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T05:30:30,896 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-06T05:30:30,896 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/nfs.dump.dir in system properties and HBase conf
2024-12-06T05:30:30,896 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/java.io.tmpdir in system properties and HBase conf
2024-12-06T05:30:30,896 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T05:30:30,896 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-06T05:30:30,896 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-06T05:30:30,912 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T05:30:30,932 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:31,086 INFO  [regionserver/ec46afd49254:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T05:30:31,213 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:30:31,216 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:30:31,217 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:30:31,217 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:30:31,217 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T05:30:31,218 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:30:31,218 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@314e7370{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:30:31,219 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@425d5d71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:30:31,328 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5ff7780b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/java.io.tmpdir/jetty-localhost-35481-hadoop-hdfs-3_4_1-tests_jar-_-any-11556149255665402038/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T05:30:31,328 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@20d96a0d{HTTP/1.1, (http/1.1)}{localhost:35481}
2024-12-06T05:30:31,328 INFO  [Time-limited test {}] server.Server(415): Started @250939ms
2024-12-06T05:30:31,341 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T05:30:31,610 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:30:31,613 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:30:31,614 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:30:31,614 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:30:31,614 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T05:30:31,614 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2eb912ab{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:30:31,615 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f424370{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:30:31,726 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4a6e8e46{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/java.io.tmpdir/jetty-localhost-38395-hadoop-hdfs-3_4_1-tests_jar-_-any-6481392215942641131/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:30:31,726 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1204fb24{HTTP/1.1, (http/1.1)}{localhost:38395}
2024-12-06T05:30:31,726 INFO  [Time-limited test {}] server.Server(415): Started @251337ms
2024-12-06T05:30:31,728 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:30:31,757 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:30:31,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:31,760 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:30:31,761 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:30:31,761 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:30:31,761 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T05:30:31,762 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13ef5561{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:30:31,762 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f61588{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:30:31,860 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@42c5c09{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/java.io.tmpdir/jetty-localhost-41209-hadoop-hdfs-3_4_1-tests_jar-_-any-13411291938471150952/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:30:31,861 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5ce0a24{HTTP/1.1, (http/1.1)}{localhost:41209}
2024-12-06T05:30:31,861 INFO  [Time-limited test {}] server.Server(415): Started @251472ms
2024-12-06T05:30:31,862 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:30:31,933 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:32,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:32,885 WARN  [Thread-1968 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/cluster_aa9f0358-d841-213d-3fe9-e6bb995c275f/data/data2/current/BP-652329374-172.17.0.3-1733463030916/current, will proceed with Du for space computation calculation, 
2024-12-06T05:30:32,885 WARN  [Thread-1967 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/cluster_aa9f0358-d841-213d-3fe9-e6bb995c275f/data/data1/current/BP-652329374-172.17.0.3-1733463030916/current, will proceed with Du for space computation calculation, 
2024-12-06T05:30:32,902 WARN  [Thread-1931 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:30:32,905 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x13ee0be34793e594 with lease ID 0x7bc928117241e659: Processing first storage report for DS-f13ef75b-c1b3-434f-8e76-a8d68cff2821 from datanode DatanodeRegistration(127.0.0.1:40111, datanodeUuid=d7e846d0-9983-4fd8-93fc-fc883961243f, infoPort=35415, infoSecurePort=0, ipcPort=33461, storageInfo=lv=-57;cid=testClusterID;nsid=1155093202;c=1733463030916)
2024-12-06T05:30:32,905 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x13ee0be34793e594 with lease ID 0x7bc928117241e659: from storage DS-f13ef75b-c1b3-434f-8e76-a8d68cff2821 node DatanodeRegistration(127.0.0.1:40111, datanodeUuid=d7e846d0-9983-4fd8-93fc-fc883961243f, infoPort=35415, infoSecurePort=0, ipcPort=33461, storageInfo=lv=-57;cid=testClusterID;nsid=1155093202;c=1733463030916), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:30:32,905 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x13ee0be34793e594 with lease ID 0x7bc928117241e659: Processing first storage report for DS-a2d2e7d5-a564-4bff-a7cb-160678bfa872 from datanode DatanodeRegistration(127.0.0.1:40111, datanodeUuid=d7e846d0-9983-4fd8-93fc-fc883961243f, infoPort=35415, infoSecurePort=0, ipcPort=33461, storageInfo=lv=-57;cid=testClusterID;nsid=1155093202;c=1733463030916)
2024-12-06T05:30:32,905 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x13ee0be34793e594 with lease ID 0x7bc928117241e659: from storage DS-a2d2e7d5-a564-4bff-a7cb-160678bfa872 node DatanodeRegistration(127.0.0.1:40111, datanodeUuid=d7e846d0-9983-4fd8-93fc-fc883961243f, infoPort=35415, infoSecurePort=0, ipcPort=33461, storageInfo=lv=-57;cid=testClusterID;nsid=1155093202;c=1733463030916), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:30:32,934 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:33,030 WARN  [Thread-1978 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/cluster_aa9f0358-d841-213d-3fe9-e6bb995c275f/data/data3/current/BP-652329374-172.17.0.3-1733463030916/current, will proceed with Du for space computation calculation, 
2024-12-06T05:30:33,030 WARN  [Thread-1979 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/cluster_aa9f0358-d841-213d-3fe9-e6bb995c275f/data/data4/current/BP-652329374-172.17.0.3-1733463030916/current, will proceed with Du for space computation calculation, 
2024-12-06T05:30:33,057 WARN  [Thread-1954 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:30:33,059 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84e969294c1b6ee7 with lease ID 0x7bc928117241e65a: Processing first storage report for DS-c90bbfcc-4761-4ddc-afba-05fa8639e0ec from datanode DatanodeRegistration(127.0.0.1:41663, datanodeUuid=83a570a5-e71b-4269-9188-b090b003c8e9, infoPort=36837, infoSecurePort=0, ipcPort=41307, storageInfo=lv=-57;cid=testClusterID;nsid=1155093202;c=1733463030916)
2024-12-06T05:30:33,059 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84e969294c1b6ee7 with lease ID 0x7bc928117241e65a: from storage DS-c90bbfcc-4761-4ddc-afba-05fa8639e0ec node DatanodeRegistration(127.0.0.1:41663, datanodeUuid=83a570a5-e71b-4269-9188-b090b003c8e9, infoPort=36837, infoSecurePort=0, ipcPort=41307, storageInfo=lv=-57;cid=testClusterID;nsid=1155093202;c=1733463030916), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:30:33,060 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84e969294c1b6ee7 with lease ID 0x7bc928117241e65a: Processing first storage report for DS-421556ce-1bf7-4216-8dfa-ac2c1e80709e from datanode DatanodeRegistration(127.0.0.1:41663, datanodeUuid=83a570a5-e71b-4269-9188-b090b003c8e9, infoPort=36837, infoSecurePort=0, ipcPort=41307, storageInfo=lv=-57;cid=testClusterID;nsid=1155093202;c=1733463030916)
2024-12-06T05:30:33,060 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84e969294c1b6ee7 with lease ID 0x7bc928117241e65a: from storage DS-421556ce-1bf7-4216-8dfa-ac2c1e80709e node DatanodeRegistration(127.0.0.1:41663, datanodeUuid=83a570a5-e71b-4269-9188-b090b003c8e9, infoPort=36837, infoSecurePort=0, ipcPort=41307, storageInfo=lv=-57;cid=testClusterID;nsid=1155093202;c=1733463030916), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:30:33,093 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe
2024-12-06T05:30:33,097 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/cluster_aa9f0358-d841-213d-3fe9-e6bb995c275f/zookeeper_0, clientPort=57049, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/cluster_aa9f0358-d841-213d-3fe9-e6bb995c275f/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/cluster_aa9f0358-d841-213d-3fe9-e6bb995c275f/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-06T05:30:33,098 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57049
2024-12-06T05:30:33,098 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:30:33,099 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:30:33,110 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741825_1001 (size=7)
2024-12-06T05:30:33,111 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741825_1001 (size=7)
2024-12-06T05:30:33,112 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da with version=8
2024-12-06T05:30:33,112 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/hbase-staging
2024-12-06T05:30:33,114 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/ec46afd49254:0 server-side Connection retries=45
2024-12-06T05:30:33,115 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:30:33,115 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T05:30:33,115 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T05:30:33,115 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:30:33,115 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T05:30:33,115 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-06T05:30:33,115 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T05:30:33,116 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44883
2024-12-06T05:30:33,117 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44883 connecting to ZooKeeper ensemble=127.0.0.1:57049
2024-12-06T05:30:33,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:448830x0, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T05:30:33,177 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44883-0x101a91ec0b30000 connected
2024-12-06T05:30:33,261 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:30:33,263 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:30:33,265 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:30:33,266 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da, hbase.cluster.distributed=false
2024-12-06T05:30:33,267 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T05:30:33,268 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44883
2024-12-06T05:30:33,274 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44883
2024-12-06T05:30:33,275 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44883
2024-12-06T05:30:33,275 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44883
2024-12-06T05:30:33,275 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44883
2024-12-06T05:30:33,291 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/ec46afd49254:0 server-side Connection retries=45
2024-12-06T05:30:33,292 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:30:33,292 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T05:30:33,292 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T05:30:33,292 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:30:33,292 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T05:30:33,292 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-06T05:30:33,292 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T05:30:33,293 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42957
2024-12-06T05:30:33,294 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42957 connecting to ZooKeeper ensemble=127.0.0.1:57049
2024-12-06T05:30:33,295 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:30:33,297 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:30:33,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:429570x0, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T05:30:33,303 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:429570x0, quorum=127.0.0.1:57049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:30:33,303 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42957-0x101a91ec0b30001 connected
2024-12-06T05:30:33,303 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-06T05:30:33,304 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-06T05:30:33,304 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-06T05:30:33,305 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T05:30:33,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42957
2024-12-06T05:30:33,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42957
2024-12-06T05:30:33,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42957
2024-12-06T05:30:33,308 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42957
2024-12-06T05:30:33,308 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42957
2024-12-06T05:30:33,320 DEBUG [M:0;ec46afd49254:44883 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ec46afd49254:44883
2024-12-06T05:30:33,320 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ec46afd49254,44883,1733463033114
2024-12-06T05:30:33,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:30:33,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:30:33,324 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ec46afd49254,44883,1733463033114
2024-12-06T05:30:33,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-06T05:30:33,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:30:33,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:30:33,335 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-06T05:30:33,335 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ec46afd49254,44883,1733463033114 from backup master directory
2024-12-06T05:30:33,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ec46afd49254,44883,1733463033114
2024-12-06T05:30:33,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:30:33,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:30:33,345 WARN  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T05:30:33,345 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ec46afd49254,44883,1733463033114
2024-12-06T05:30:33,349 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/hbase.id] with ID: 208fc6a9-1640-4dca-80ed-0a228c3dff48
2024-12-06T05:30:33,349 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/.tmp/hbase.id
2024-12-06T05:30:33,357 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741826_1002 (size=42)
2024-12-06T05:30:33,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741826_1002 (size=42)
2024-12-06T05:30:33,358 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/.tmp/hbase.id]:[hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/hbase.id]
2024-12-06T05:30:33,369 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:30:33,369 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-06T05:30:33,370 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-06T05:30:33,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:30:33,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:30:33,390 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741827_1003 (size=196)
2024-12-06T05:30:33,391 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741827_1003 (size=196)
2024-12-06T05:30:33,391 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T05:30:33,392 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-06T05:30:33,392 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:30:33,399 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741828_1004 (size=1189)
2024-12-06T05:30:33,399 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741828_1004 (size=1189)
2024-12-06T05:30:33,400 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store
2024-12-06T05:30:33,407 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741829_1005 (size=34)
2024-12-06T05:30:33,407 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741829_1005 (size=34)
2024-12-06T05:30:33,407 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:30:33,408 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T05:30:33,408 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:30:33,408 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:30:33,408 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T05:30:33,408 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:30:33,408 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:30:33,408 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733463033408Disabling compacts and flushes for region at 1733463033408Disabling writes for close at 1733463033408Writing region close event to WAL at 1733463033408Closed at 1733463033408
2024-12-06T05:30:33,409 WARN  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/.initializing
2024-12-06T05:30:33,409 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/WALs/ec46afd49254,44883,1733463033114
2024-12-06T05:30:33,411 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C44883%2C1733463033114, suffix=, logDir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/WALs/ec46afd49254,44883,1733463033114, archiveDir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/oldWALs, maxLogs=10
2024-12-06T05:30:33,412 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C44883%2C1733463033114.1733463033411
2024-12-06T05:30:33,416 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/WALs/ec46afd49254,44883,1733463033114/ec46afd49254%2C44883%2C1733463033114.1733463033411
2024-12-06T05:30:33,417 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36837:36837),(127.0.0.1/127.0.0.1:35415:35415)]
2024-12-06T05:30:33,420 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:30:33,421 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:30:33,421 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:30:33,421 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:30:33,422 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:30:33,423 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-06T05:30:33,423 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:33,424 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:30:33,424 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:30:33,425 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-06T05:30:33,425 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:33,426 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:30:33,426 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:30:33,427 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-06T05:30:33,427 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:33,427 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:30:33,427 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:30:33,429 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-06T05:30:33,429 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:33,429 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:30:33,429 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:30:33,430 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:30:33,430 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:30:33,432 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:30:33,432 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:30:33,432 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-06T05:30:33,433 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:30:33,437 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:30:33,437 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=854987, jitterRate=0.08717341721057892}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-06T05:30:33,438 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733463033421Initializing all the Stores at 1733463033422 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463033422Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463033422Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463033422Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463033422Cleaning up temporary data from old regions at 1733463033432 (+10 ms)Region opened successfully at 1733463033438 (+6 ms)
2024-12-06T05:30:33,438 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-06T05:30:33,441 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69cc6f30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0
2024-12-06T05:30:33,442 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-06T05:30:33,442 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-06T05:30:33,442 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-06T05:30:33,442 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-06T05:30:33,443 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-06T05:30:33,443 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-06T05:30:33,443 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-06T05:30:33,445 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-06T05:30:33,446 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-06T05:30:33,450 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-06T05:30:33,450 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-06T05:30:33,451 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-06T05:30:33,460 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-06T05:30:33,461 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-06T05:30:33,462 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-06T05:30:33,471 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-06T05:30:33,472 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-06T05:30:33,481 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-06T05:30:33,483 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-06T05:30:33,492 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-06T05:30:33,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T05:30:33,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T05:30:33,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:30:33,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:30:33,503 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ec46afd49254,44883,1733463033114, sessionid=0x101a91ec0b30000, setting cluster-up flag (Was=false)
2024-12-06T05:30:33,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:30:33,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:30:33,555 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-06T05:30:33,556 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec46afd49254,44883,1733463033114
2024-12-06T05:30:33,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:30:33,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:30:33,608 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-06T05:30:33,609 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec46afd49254,44883,1733463033114
2024-12-06T05:30:33,610 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-06T05:30:33,611 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-06T05:30:33,611 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-06T05:30:33,612 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-06T05:30:33,612 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ec46afd49254,44883,1733463033114
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-06T05:30:33,613 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:30:33,613 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:30:33,613 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:30:33,613 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:30:33,613 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ec46afd49254:0, corePoolSize=10, maxPoolSize=10
2024-12-06T05:30:33,613 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:30:33,613 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ec46afd49254:0, corePoolSize=2, maxPoolSize=2
2024-12-06T05:30:33,613 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:30:33,614 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733463063614
2024-12-06T05:30:33,614 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-06T05:30:33,614 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-06T05:30:33,614 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-06T05:30:33,614 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-06T05:30:33,614 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-06T05:30:33,614 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-06T05:30:33,615 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:33,615 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:30:33,615 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-06T05:30:33,615 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-06T05:30:33,615 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-06T05:30:33,615 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-06T05:30:33,615 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-06T05:30:33,615 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-06T05:30:33,616 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733463033616,5,FailOnTimeoutGroup]
2024-12-06T05:30:33,616 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733463033616,5,FailOnTimeoutGroup]
2024-12-06T05:30:33,616 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:33,616 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-06T05:30:33,616 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:33,616 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:33,616 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:33,616 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-06T05:30:33,622 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741831_1007 (size=1321)
2024-12-06T05:30:33,622 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741831_1007 (size=1321)
2024-12-06T05:30:33,623 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-06T05:30:33,623 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da
2024-12-06T05:30:33,628 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741832_1008 (size=32)
2024-12-06T05:30:33,629 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:30:33,630 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741832_1008 (size=32)
2024-12-06T05:30:33,630 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T05:30:33,631 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T05:30:33,631 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:33,632 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:30:33,632 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T05:30:33,634 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T05:30:33,634 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:33,634 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:30:33,634 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T05:30:33,636 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T05:30:33,636 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:33,636 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:30:33,636 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T05:30:33,637 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T05:30:33,637 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:33,638 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:30:33,638 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T05:30:33,639 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740
2024-12-06T05:30:33,639 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740
2024-12-06T05:30:33,640 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T05:30:33,640 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T05:30:33,641 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T05:30:33,642 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T05:30:33,644 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:30:33,644 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=836753, jitterRate=0.06398648023605347}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T05:30:33,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733463033629Initializing all the Stores at 1733463033630 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463033630Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463033630Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463033630Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463033630Cleaning up temporary data from old regions at 1733463033640 (+10 ms)Region opened successfully at 1733463033645 (+5 ms)
2024-12-06T05:30:33,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T05:30:33,645 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T05:30:33,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T05:30:33,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T05:30:33,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T05:30:33,646 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T05:30:33,646 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733463033645Disabling compacts and flushes for region at 1733463033645Disabling writes for close at 1733463033645Writing region close event to WAL at 1733463033646 (+1 ms)Closed at 1733463033646
2024-12-06T05:30:33,647 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:30:33,647 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-06T05:30:33,647 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-06T05:30:33,649 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T05:30:33,650 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-06T05:30:33,710 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer(746): ClusterId : 208fc6a9-1640-4dca-80ed-0a228c3dff48
2024-12-06T05:30:33,710 DEBUG [RS:0;ec46afd49254:42957 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-06T05:30:33,724 DEBUG [RS:0;ec46afd49254:42957 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-06T05:30:33,724 DEBUG [RS:0;ec46afd49254:42957 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-06T05:30:33,735 DEBUG [RS:0;ec46afd49254:42957 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-06T05:30:33,735 DEBUG [RS:0;ec46afd49254:42957 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c39e47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0
2024-12-06T05:30:33,746 DEBUG [RS:0;ec46afd49254:42957 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ec46afd49254:42957
2024-12-06T05:30:33,746 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-06T05:30:33,746 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-06T05:30:33,746 DEBUG [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-06T05:30:33,747 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec46afd49254,44883,1733463033114 with port=42957, startcode=1733463033291
2024-12-06T05:30:33,747 DEBUG [RS:0;ec46afd49254:42957 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-06T05:30:33,749 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57571, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-12-06T05:30:33,749 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44883 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ec46afd49254,42957,1733463033291
2024-12-06T05:30:33,750 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44883 {}] master.ServerManager(517): Registering regionserver=ec46afd49254,42957,1733463033291
2024-12-06T05:30:33,751 DEBUG [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da
2024-12-06T05:30:33,751 DEBUG [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33745
2024-12-06T05:30:33,751 DEBUG [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-06T05:30:33,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T05:30:33,755 DEBUG [RS:0;ec46afd49254:42957 {}] zookeeper.ZKUtil(111): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec46afd49254,42957,1733463033291
2024-12-06T05:30:33,755 WARN  [RS:0;ec46afd49254:42957 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T05:30:33,756 INFO  [RS:0;ec46afd49254:42957 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:30:33,756 DEBUG [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/WALs/ec46afd49254,42957,1733463033291
2024-12-06T05:30:33,756 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec46afd49254,42957,1733463033291]
2024-12-06T05:30:33,759 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-06T05:30:33,760 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:33,764 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-06T05:30:33,764 INFO  [RS:0;ec46afd49254:42957 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T05:30:33,764 INFO  [RS:0;ec46afd49254:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:33,771 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-06T05:30:33,772 INFO  [RS:0;ec46afd49254:42957 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-06T05:30:33,773 INFO  [RS:0;ec46afd49254:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:33,773 DEBUG [RS:0;ec46afd49254:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:30:33,773 DEBUG [RS:0;ec46afd49254:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:30:33,773 DEBUG [RS:0;ec46afd49254:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:30:33,773 DEBUG [RS:0;ec46afd49254:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:30:33,773 DEBUG [RS:0;ec46afd49254:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:30:33,773 DEBUG [RS:0;ec46afd49254:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0, corePoolSize=2, maxPoolSize=2
2024-12-06T05:30:33,773 DEBUG [RS:0;ec46afd49254:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:30:33,773 DEBUG [RS:0;ec46afd49254:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:30:33,773 DEBUG [RS:0;ec46afd49254:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:30:33,773 DEBUG [RS:0;ec46afd49254:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:30:33,773 DEBUG [RS:0;ec46afd49254:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:30:33,773 DEBUG [RS:0;ec46afd49254:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:30:33,773 DEBUG [RS:0;ec46afd49254:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3
2024-12-06T05:30:33,773 DEBUG [RS:0;ec46afd49254:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3
2024-12-06T05:30:33,774 INFO  [RS:0;ec46afd49254:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:33,774 INFO  [RS:0;ec46afd49254:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:33,774 INFO  [RS:0;ec46afd49254:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:33,774 INFO  [RS:0;ec46afd49254:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:33,774 INFO  [RS:0;ec46afd49254:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:33,774 INFO  [RS:0;ec46afd49254:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,42957,1733463033291-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T05:30:33,789 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-06T05:30:33,789 INFO  [RS:0;ec46afd49254:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,42957,1733463033291-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:33,790 INFO  [RS:0;ec46afd49254:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:33,790 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.Replication(171): ec46afd49254,42957,1733463033291 started
2024-12-06T05:30:33,800 WARN  [ec46afd49254:44883 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-06T05:30:33,805 INFO  [RS:0;ec46afd49254:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:33,805 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer(1482): Serving as ec46afd49254,42957,1733463033291, RpcServer on ec46afd49254/172.17.0.3:42957, sessionid=0x101a91ec0b30001
2024-12-06T05:30:33,805 DEBUG [RS:0;ec46afd49254:42957 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-06T05:30:33,805 DEBUG [RS:0;ec46afd49254:42957 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec46afd49254,42957,1733463033291
2024-12-06T05:30:33,805 DEBUG [RS:0;ec46afd49254:42957 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,42957,1733463033291'
2024-12-06T05:30:33,805 DEBUG [RS:0;ec46afd49254:42957 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-06T05:30:33,806 DEBUG [RS:0;ec46afd49254:42957 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-06T05:30:33,806 DEBUG [RS:0;ec46afd49254:42957 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-06T05:30:33,806 DEBUG [RS:0;ec46afd49254:42957 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-06T05:30:33,806 DEBUG [RS:0;ec46afd49254:42957 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec46afd49254,42957,1733463033291
2024-12-06T05:30:33,806 DEBUG [RS:0;ec46afd49254:42957 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,42957,1733463033291'
2024-12-06T05:30:33,806 DEBUG [RS:0;ec46afd49254:42957 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-06T05:30:33,806 DEBUG [RS:0;ec46afd49254:42957 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-06T05:30:33,807 DEBUG [RS:0;ec46afd49254:42957 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-06T05:30:33,807 INFO  [RS:0;ec46afd49254:42957 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-06T05:30:33,807 INFO  [RS:0;ec46afd49254:42957 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-06T05:30:33,909 INFO  [RS:0;ec46afd49254:42957 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C42957%2C1733463033291, suffix=, logDir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/WALs/ec46afd49254,42957,1733463033291, archiveDir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/oldWALs, maxLogs=32
2024-12-06T05:30:33,909 INFO  [RS:0;ec46afd49254:42957 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C42957%2C1733463033291.1733463033909
2024-12-06T05:30:33,919 INFO  [RS:0;ec46afd49254:42957 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/WALs/ec46afd49254,42957,1733463033291/ec46afd49254%2C42957%2C1733463033291.1733463033909
2024-12-06T05:30:33,920 DEBUG [RS:0;ec46afd49254:42957 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36837:36837),(127.0.0.1/127.0.0.1:35415:35415)]
2024-12-06T05:30:33,934 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:34,051 DEBUG [ec46afd49254:44883 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-06T05:30:34,051 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ec46afd49254,42957,1733463033291
2024-12-06T05:30:34,052 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec46afd49254,42957,1733463033291, state=OPENING
2024-12-06T05:30:34,060 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-06T05:30:34,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:30:34,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:30:34,072 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T05:30:34,072 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:30:34,072 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:30:34,072 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ec46afd49254,42957,1733463033291}]
2024-12-06T05:30:34,225 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-06T05:30:34,227 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33185, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-06T05:30:34,231 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-06T05:30:34,231 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:30:34,233 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C42957%2C1733463033291.meta, suffix=.meta, logDir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/WALs/ec46afd49254,42957,1733463033291, archiveDir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/oldWALs, maxLogs=32
2024-12-06T05:30:34,234 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C42957%2C1733463033291.meta.1733463034233.meta
2024-12-06T05:30:34,239 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/WALs/ec46afd49254,42957,1733463033291/ec46afd49254%2C42957%2C1733463033291.meta.1733463034233.meta
2024-12-06T05:30:34,239 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35415:35415),(127.0.0.1/127.0.0.1:36837:36837)]
2024-12-06T05:30:34,247 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:30:34,247 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-06T05:30:34,248 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-06T05:30:34,248 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-06T05:30:34,248 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-06T05:30:34,248 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:30:34,248 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-06T05:30:34,248 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-06T05:30:34,250 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T05:30:34,251 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T05:30:34,251 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:34,251 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:30:34,251 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T05:30:34,252 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T05:30:34,252 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:34,253 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:30:34,253 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T05:30:34,254 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T05:30:34,254 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:34,254 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:30:34,254 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T05:30:34,255 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T05:30:34,255 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:34,255 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:30:34,255 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T05:30:34,256 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740
2024-12-06T05:30:34,257 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740
2024-12-06T05:30:34,258 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T05:30:34,258 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T05:30:34,259 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T05:30:34,260 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T05:30:34,261 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=767845, jitterRate=-0.023634687066078186}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T05:30:34,261 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-06T05:30:34,261 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733463034248Writing region info on filesystem at 1733463034248Initializing all the Stores at 1733463034249 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463034249Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463034249Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463034249Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463034249Cleaning up temporary data from old regions at 1733463034258 (+9 ms)Running coprocessor post-open hooks at 1733463034261 (+3 ms)Region opened successfully at 1733463034261
2024-12-06T05:30:34,262 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733463034225
2024-12-06T05:30:34,264 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-06T05:30:34,264 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-06T05:30:34,265 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ec46afd49254,42957,1733463033291
2024-12-06T05:30:34,266 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec46afd49254,42957,1733463033291, state=OPEN
2024-12-06T05:30:34,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T05:30:34,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T05:30:34,341 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:30:34,341 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:30:34,341 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ec46afd49254,42957,1733463033291
2024-12-06T05:30:34,344 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-06T05:30:34,344 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec46afd49254,42957,1733463033291 in 269 msec
2024-12-06T05:30:34,347 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-06T05:30:34,347 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 697 msec
2024-12-06T05:30:34,348 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:30:34,348 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-06T05:30:34,349 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T05:30:34,349 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec46afd49254,42957,1733463033291, seqNum=-1]
2024-12-06T05:30:34,350 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T05:30:34,351 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50143, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T05:30:34,356 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 744 msec
2024-12-06T05:30:34,356 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733463034356, completionTime=-1
2024-12-06T05:30:34,356 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-06T05:30:34,356 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-06T05:30:34,358 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-06T05:30:34,358 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733463094358
2024-12-06T05:30:34,358 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733463154358
2024-12-06T05:30:34,359 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-06T05:30:34,359 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,44883,1733463033114-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:34,359 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,44883,1733463033114-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:34,359 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,44883,1733463033114-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:34,359 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ec46afd49254:44883, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:34,359 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:34,359 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:34,361 DEBUG [master/ec46afd49254:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T05:30:34,363 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.018sec
2024-12-06T05:30:34,364 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-06T05:30:34,364 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-06T05:30:34,364 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-06T05:30:34,364 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-06T05:30:34,364 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-06T05:30:34,364 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,44883,1733463033114-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T05:30:34,364 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,44883,1733463033114-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-06T05:30:34,366 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-06T05:30:34,367 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-06T05:30:34,367 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,44883,1733463033114-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:30:34,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27df3092, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:30:34,411 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ec46afd49254,44883,-1 for getting cluster id
2024-12-06T05:30:34,411 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-06T05:30:34,412 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '208fc6a9-1640-4dca-80ed-0a228c3dff48'
2024-12-06T05:30:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-06T05:30:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "208fc6a9-1640-4dca-80ed-0a228c3dff48"

2024-12-06T05:30:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc61e48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:30:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ec46afd49254,44883,-1]
2024-12-06T05:30:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-06T05:30:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:30:34,415 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-06T05:30:34,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63515f8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:30:34,416 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T05:30:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec46afd49254,42957,1733463033291, seqNum=-1]
2024-12-06T05:30:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T05:30:34,419 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T05:30:34,422 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ec46afd49254,44883,1733463033114
2024-12-06T05:30:34,422 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:30:34,426 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-06T05:30:34,426 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-06T05:30:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is ec46afd49254,44883,1733463033114
2024-12-06T05:30:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3f98ea4a
2024-12-06T05:30:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-06T05:30:34,429 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-06T05:30:34,429 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44883 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-06T05:30:34,429 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44883 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-06T05:30:34,430 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44883 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T05:30:34,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44883 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-12-06T05:30:34,433 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-06T05:30:34,433 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:34,433 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44883 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-12-06T05:30:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T05:30:34,434 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-06T05:30:34,440 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741835_1011 (size=381)
2024-12-06T05:30:34,440 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741835_1011 (size=381)
2024-12-06T05:30:34,442 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5206502d6b4766aaa207f7e6fc8fc803, NAME => 'TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da
2024-12-06T05:30:34,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741836_1012 (size=64)
2024-12-06T05:30:34,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741836_1012 (size=64)
2024-12-06T05:30:34,448 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:30:34,448 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 5206502d6b4766aaa207f7e6fc8fc803, disabling compactions & flushes
2024-12-06T05:30:34,448 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.
2024-12-06T05:30:34,448 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.
2024-12-06T05:30:34,448 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803. after waiting 0 ms
2024-12-06T05:30:34,448 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.
2024-12-06T05:30:34,448 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.
2024-12-06T05:30:34,448 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5206502d6b4766aaa207f7e6fc8fc803:
Waiting for close lock at 1733463034448Disabling compacts and flushes for region at 1733463034448Disabling writes for close at 1733463034448Writing region close event to WAL at 1733463034448Closed at 1733463034448
2024-12-06T05:30:34,450 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-06T05:30:34,450 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733463034450"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463034450"}]},"ts":"1733463034450"}
2024-12-06T05:30:34,452 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-06T05:30:34,453 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-06T05:30:34,453 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733463034453"}]},"ts":"1733463034453"}
2024-12-06T05:30:34,455 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-12-06T05:30:34,456 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5206502d6b4766aaa207f7e6fc8fc803, ASSIGN}]
2024-12-06T05:30:34,457 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5206502d6b4766aaa207f7e6fc8fc803, ASSIGN
2024-12-06T05:30:34,458 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5206502d6b4766aaa207f7e6fc8fc803, ASSIGN; state=OFFLINE, location=ec46afd49254,42957,1733463033291; forceNewPlan=false, retain=false
2024-12-06T05:30:34,609 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5206502d6b4766aaa207f7e6fc8fc803, regionState=OPENING, regionLocation=ec46afd49254,42957,1733463033291
2024-12-06T05:30:34,612 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5206502d6b4766aaa207f7e6fc8fc803, ASSIGN because future has completed
2024-12-06T05:30:34,613 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5206502d6b4766aaa207f7e6fc8fc803, server=ec46afd49254,42957,1733463033291}]
2024-12-06T05:30:34,761 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:34,771 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.
2024-12-06T05:30:34,771 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5206502d6b4766aaa207f7e6fc8fc803, NAME => 'TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:30:34,772 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:34,772 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:30:34,772 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:34,772 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:34,774 INFO  [StoreOpener-5206502d6b4766aaa207f7e6fc8fc803-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 5206502d6b4766aaa207f7e6fc8fc803 
2024-12-06T05:30:34,775 INFO  [StoreOpener-5206502d6b4766aaa207f7e6fc8fc803-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5206502d6b4766aaa207f7e6fc8fc803 columnFamilyName info
2024-12-06T05:30:34,775 DEBUG [StoreOpener-5206502d6b4766aaa207f7e6fc8fc803-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:34,776 INFO  [StoreOpener-5206502d6b4766aaa207f7e6fc8fc803-1 {}] regionserver.HStore(327): Store=5206502d6b4766aaa207f7e6fc8fc803/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:30:34,776 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:34,777 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:34,777 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:34,778 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:34,778 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:34,780 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:34,782 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:30:34,782 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5206502d6b4766aaa207f7e6fc8fc803; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=770434, jitterRate=-0.0203436017036438}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-06T05:30:34,783 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:34,783 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5206502d6b4766aaa207f7e6fc8fc803:
Running coprocessor pre-open hook at 1733463034772Writing region info on filesystem at 1733463034772Initializing all the Stores at 1733463034773 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463034773Cleaning up temporary data from old regions at 1733463034778 (+5 ms)Running coprocessor post-open hooks at 1733463034783 (+5 ms)Region opened successfully at 1733463034783
2024-12-06T05:30:34,784 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803., pid=6, masterSystemTime=1733463034767
2024-12-06T05:30:34,786 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.
2024-12-06T05:30:34,786 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.
2024-12-06T05:30:34,787 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5206502d6b4766aaa207f7e6fc8fc803, regionState=OPEN, openSeqNum=2, regionLocation=ec46afd49254,42957,1733463033291
2024-12-06T05:30:34,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5206502d6b4766aaa207f7e6fc8fc803, server=ec46afd49254,42957,1733463033291 because future has completed
2024-12-06T05:30:34,793 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-06T05:30:34,794 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5206502d6b4766aaa207f7e6fc8fc803, server=ec46afd49254,42957,1733463033291 in 178 msec
2024-12-06T05:30:34,796 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-06T05:30:34,796 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5206502d6b4766aaa207f7e6fc8fc803, ASSIGN in 338 msec
2024-12-06T05:30:34,797 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-06T05:30:34,798 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733463034797"}]},"ts":"1733463034797"}
2024-12-06T05:30:34,800 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-12-06T05:30:34,801 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-06T05:30:34,803 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 371 msec
2024-12-06T05:30:34,935 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:35,338 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,338 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,338 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,339 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,339 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,339 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,339 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,340 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,359 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,359 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,359 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,360 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,360 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,360 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,362 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,362 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,363 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,364 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,761 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:35,869 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T05:30:35,870 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,870 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,870 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,871 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,871 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,871 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,872 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,873 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,891 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,892 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,892 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,892 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,892 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,893 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,897 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,897 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,897 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,900 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:35,936 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:36,505 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-06T05:30:36,505 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-06T05:30:36,506 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T05:30:36,762 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:36,936 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:37,763 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:37,937 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:38,763 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:38,938 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:39,759 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-06T05:30:39,760 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-12-06T05:30:39,764 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:39,938 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:40,765 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:40,939 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:41,765 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:41,940 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:42,010 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T05:30:42,011 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,011 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,011 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,012 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,012 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,012 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,013 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,013 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,036 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,037 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,037 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,037 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,037 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,038 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,041 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,041 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,041 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,044 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:42,766 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:42,941 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:43,767 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:43,941 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T05:30:44,457 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-12-06T05:30:44,457 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-12-06T05:30:44,461 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-12-06T05:30:44,461 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.
2024-12-06T05:30:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803., hostname=ec46afd49254,42957,1733463033291, seqNum=2]
2024-12-06T05:30:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:44,479 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5206502d6b4766aaa207f7e6fc8fc803 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T05:30:44,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/42801156f20f4a41a6efbfcf128ee6f3 is 1080, key is row0001/info:/1733463044467/Put/seqid=0
2024-12-06T05:30:44,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741837_1013 (size=12509)
2024-12-06T05:30:44,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741837_1013 (size=12509)
2024-12-06T05:30:44,506 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/42801156f20f4a41a6efbfcf128ee6f3
2024-12-06T05:30:44,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/42801156f20f4a41a6efbfcf128ee6f3 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/42801156f20f4a41a6efbfcf128ee6f3
2024-12-06T05:30:44,519 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/42801156f20f4a41a6efbfcf128ee6f3, entries=7, sequenceid=11, filesize=12.2 K
2024-12-06T05:30:44,520 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=19.96 KB/20444 for 5206502d6b4766aaa207f7e6fc8fc803 in 41ms, sequenceid=11, compaction requested=false
2024-12-06T05:30:44,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5206502d6b4766aaa207f7e6fc8fc803:

2024-12-06T05:30:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:44,522 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5206502d6b4766aaa207f7e6fc8fc803 1/1 column families, dataSize=21.02 KB heapSize=22.75 KB
2024-12-06T05:30:44,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/3420dcab666541d4bf1908f74a80e8fe is 1080, key is row0008/info:/1733463044481/Put/seqid=0
2024-12-06T05:30:44,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741838_1014 (size=26530)
2024-12-06T05:30:44,542 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741838_1014 (size=26530)
2024-12-06T05:30:44,542 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/3420dcab666541d4bf1908f74a80e8fe
2024-12-06T05:30:44,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/3420dcab666541d4bf1908f74a80e8fe as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/3420dcab666541d4bf1908f74a80e8fe
2024-12-06T05:30:44,553 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/3420dcab666541d4bf1908f74a80e8fe, entries=20, sequenceid=34, filesize=25.9 K
2024-12-06T05:30:44,554 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~21.02 KB/21520, heapSize ~22.73 KB/23280, currentSize=5.25 KB/5380 for 5206502d6b4766aaa207f7e6fc8fc803 in 33ms, sequenceid=34, compaction requested=false
2024-12-06T05:30:44,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5206502d6b4766aaa207f7e6fc8fc803:

2024-12-06T05:30:44,554 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=38.1 K, sizeToCheck=16.0 K
2024-12-06T05:30:44,555 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:30:44,555 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/3420dcab666541d4bf1908f74a80e8fe because midkey is the same as first or last row
2024-12-06T05:30:44,767 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:44,942 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:45,768 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:45,943 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:46,537 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5206502d6b4766aaa207f7e6fc8fc803 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T05:30:46,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/961c8f9bab2946d3b738b924e191a092 is 1080, key is row0028/info:/1733463044523/Put/seqid=0
2024-12-06T05:30:46,549 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741839_1015 (size=12509)
2024-12-06T05:30:46,549 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741839_1015 (size=12509)
2024-12-06T05:30:46,551 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/961c8f9bab2946d3b738b924e191a092
2024-12-06T05:30:46,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/961c8f9bab2946d3b738b924e191a092 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/961c8f9bab2946d3b738b924e191a092
2024-12-06T05:30:46,565 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/961c8f9bab2946d3b738b924e191a092, entries=7, sequenceid=44, filesize=12.2 K
2024-12-06T05:30:46,566 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 5206502d6b4766aaa207f7e6fc8fc803 in 28ms, sequenceid=44, compaction requested=true
2024-12-06T05:30:46,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5206502d6b4766aaa207f7e6fc8fc803:

2024-12-06T05:30:46,566 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=50.3 K, sizeToCheck=16.0 K
2024-12-06T05:30:46,566 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:30:46,566 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/3420dcab666541d4bf1908f74a80e8fe because midkey is the same as first or last row
2024-12-06T05:30:46,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5206502d6b4766aaa207f7e6fc8fc803:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T05:30:46,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:30:46,566 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T05:30:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:46,567 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5206502d6b4766aaa207f7e6fc8fc803 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-12-06T05:30:46,567 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 51548 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T05:30:46,568 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1541): 5206502d6b4766aaa207f7e6fc8fc803/info is initiating minor compaction (all files)
2024-12-06T05:30:46,568 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5206502d6b4766aaa207f7e6fc8fc803/info in TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.
2024-12-06T05:30:46,568 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/42801156f20f4a41a6efbfcf128ee6f3, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/3420dcab666541d4bf1908f74a80e8fe, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/961c8f9bab2946d3b738b924e191a092] into tmpdir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp, totalSize=50.3 K
2024-12-06T05:30:46,568 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42801156f20f4a41a6efbfcf128ee6f3, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733463044467
2024-12-06T05:30:46,569 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3420dcab666541d4bf1908f74a80e8fe, keycount=20, bloomtype=ROW, size=25.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733463044481
2024-12-06T05:30:46,569 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 961c8f9bab2946d3b738b924e191a092, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733463044523
2024-12-06T05:30:46,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/d8487465af86407ba3a81894427017e9 is 1080, key is row0035/info:/1733463046539/Put/seqid=0
2024-12-06T05:30:46,583 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741840_1016 (size=18987)
2024-12-06T05:30:46,583 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741840_1016 (size=18987)
2024-12-06T05:30:46,584 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/d8487465af86407ba3a81894427017e9
2024-12-06T05:30:46,591 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5206502d6b4766aaa207f7e6fc8fc803#info#compaction#59 average throughput is 34.89 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T05:30:46,592 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/f83fb54be9ab49f58d57e064f9e282f2 is 1080, key is row0001/info:/1733463044467/Put/seqid=0
2024-12-06T05:30:46,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/d8487465af86407ba3a81894427017e9 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/d8487465af86407ba3a81894427017e9
2024-12-06T05:30:46,598 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741841_1017 (size=41747)
2024-12-06T05:30:46,599 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741841_1017 (size=41747)
2024-12-06T05:30:46,601 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/d8487465af86407ba3a81894427017e9, entries=13, sequenceid=60, filesize=18.5 K
2024-12-06T05:30:46,602 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=13.66 KB/13988 for 5206502d6b4766aaa207f7e6fc8fc803 in 35ms, sequenceid=60, compaction requested=false
2024-12-06T05:30:46,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5206502d6b4766aaa207f7e6fc8fc803:

2024-12-06T05:30:46,603 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=68.9 K, sizeToCheck=16.0 K
2024-12-06T05:30:46,603 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:30:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:46,603 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/3420dcab666541d4bf1908f74a80e8fe because midkey is the same as first or last row
2024-12-06T05:30:46,603 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5206502d6b4766aaa207f7e6fc8fc803 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-06T05:30:46,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/18bf52f669cb402d8a2ac1cdfb3aba8f is 1080, key is row0048/info:/1733463046569/Put/seqid=0
2024-12-06T05:30:46,609 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/f83fb54be9ab49f58d57e064f9e282f2 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/f83fb54be9ab49f58d57e064f9e282f2
2024-12-06T05:30:46,614 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741842_1018 (size=20064)
2024-12-06T05:30:46,614 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741842_1018 (size=20064)
2024-12-06T05:30:46,614 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/18bf52f669cb402d8a2ac1cdfb3aba8f
2024-12-06T05:30:46,617 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5206502d6b4766aaa207f7e6fc8fc803/info of 5206502d6b4766aaa207f7e6fc8fc803 into f83fb54be9ab49f58d57e064f9e282f2(size=40.8 K), total size for store is 59.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T05:30:46,618 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5206502d6b4766aaa207f7e6fc8fc803:

2024-12-06T05:30:46,618 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803., storeName=5206502d6b4766aaa207f7e6fc8fc803/info, priority=13, startTime=1733463046566; duration=0sec
2024-12-06T05:30:46,618 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K
2024-12-06T05:30:46,618 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:30:46,618 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/f83fb54be9ab49f58d57e064f9e282f2 because midkey is the same as first or last row
2024-12-06T05:30:46,618 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K
2024-12-06T05:30:46,618 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:30:46,618 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/f83fb54be9ab49f58d57e064f9e282f2 because midkey is the same as first or last row
2024-12-06T05:30:46,618 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K
2024-12-06T05:30:46,618 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:30:46,618 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/f83fb54be9ab49f58d57e064f9e282f2 because midkey is the same as first or last row
2024-12-06T05:30:46,618 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:30:46,618 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5206502d6b4766aaa207f7e6fc8fc803:info
2024-12-06T05:30:46,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/18bf52f669cb402d8a2ac1cdfb3aba8f as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/18bf52f669cb402d8a2ac1cdfb3aba8f
2024-12-06T05:30:46,627 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/18bf52f669cb402d8a2ac1cdfb3aba8f, entries=14, sequenceid=77, filesize=19.6 K
2024-12-06T05:30:46,627 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=3.15 KB/3228 for 5206502d6b4766aaa207f7e6fc8fc803 in 24ms, sequenceid=77, compaction requested=true
2024-12-06T05:30:46,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5206502d6b4766aaa207f7e6fc8fc803:

2024-12-06T05:30:46,628 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=78.9 K, sizeToCheck=16.0 K
2024-12-06T05:30:46,628 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:30:46,628 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/f83fb54be9ab49f58d57e064f9e282f2 because midkey is the same as first or last row
2024-12-06T05:30:46,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5206502d6b4766aaa207f7e6fc8fc803:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T05:30:46,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:30:46,628 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T05:30:46,629 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 80798 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T05:30:46,629 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1541): 5206502d6b4766aaa207f7e6fc8fc803/info is initiating minor compaction (all files)
2024-12-06T05:30:46,629 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5206502d6b4766aaa207f7e6fc8fc803/info in TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.
2024-12-06T05:30:46,629 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/f83fb54be9ab49f58d57e064f9e282f2, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/d8487465af86407ba3a81894427017e9, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/18bf52f669cb402d8a2ac1cdfb3aba8f] into tmpdir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp, totalSize=78.9 K
2024-12-06T05:30:46,630 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f83fb54be9ab49f58d57e064f9e282f2, keycount=34, bloomtype=ROW, size=40.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733463044467
2024-12-06T05:30:46,630 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8487465af86407ba3a81894427017e9, keycount=13, bloomtype=ROW, size=18.5 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733463046539
2024-12-06T05:30:46,630 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18bf52f669cb402d8a2ac1cdfb3aba8f, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733463046569
2024-12-06T05:30:46,642 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5206502d6b4766aaa207f7e6fc8fc803#info#compaction#61 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T05:30:46,642 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/d2f83d55717b40aeb2b38101ca4718ae is 1080, key is row0001/info:/1733463044467/Put/seqid=0
2024-12-06T05:30:46,646 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741843_1019 (size=71001)
2024-12-06T05:30:46,647 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741843_1019 (size=71001)
2024-12-06T05:30:46,653 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/d2f83d55717b40aeb2b38101ca4718ae as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/d2f83d55717b40aeb2b38101ca4718ae
2024-12-06T05:30:46,660 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5206502d6b4766aaa207f7e6fc8fc803/info of 5206502d6b4766aaa207f7e6fc8fc803 into d2f83d55717b40aeb2b38101ca4718ae(size=69.3 K), total size for store is 69.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T05:30:46,660 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5206502d6b4766aaa207f7e6fc8fc803:

2024-12-06T05:30:46,660 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803., storeName=5206502d6b4766aaa207f7e6fc8fc803/info, priority=13, startTime=1733463046628; duration=0sec
2024-12-06T05:30:46,660 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=69.3 K, sizeToCheck=16.0 K
2024-12-06T05:30:46,660 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:30:46,660 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/d2f83d55717b40aeb2b38101ca4718ae because midkey is the same as first or last row
2024-12-06T05:30:46,660 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=69.3 K, sizeToCheck=16.0 K
2024-12-06T05:30:46,660 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:30:46,661 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/d2f83d55717b40aeb2b38101ca4718ae because midkey is the same as first or last row
2024-12-06T05:30:46,661 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=69.3 K, sizeToCheck=16.0 K
2024-12-06T05:30:46,661 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:30:46,661 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/d2f83d55717b40aeb2b38101ca4718ae because midkey is the same as first or last row
2024-12-06T05:30:46,661 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:30:46,661 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5206502d6b4766aaa207f7e6fc8fc803:info
2024-12-06T05:30:46,768 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:46,943 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:47,769 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:47,944 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:48,622 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5206502d6b4766aaa207f7e6fc8fc803 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T05:30:48,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/bdeadd37e6c349c0aa77f7f777f8ea89 is 1080, key is row0062/info:/1733463046604/Put/seqid=0
2024-12-06T05:30:48,634 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741844_1020 (size=12509)
2024-12-06T05:30:48,634 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741844_1020 (size=12509)
2024-12-06T05:30:48,635 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/bdeadd37e6c349c0aa77f7f777f8ea89
2024-12-06T05:30:48,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/bdeadd37e6c349c0aa77f7f777f8ea89 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/bdeadd37e6c349c0aa77f7f777f8ea89
2024-12-06T05:30:48,647 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/bdeadd37e6c349c0aa77f7f777f8ea89, entries=7, sequenceid=89, filesize=12.2 K
2024-12-06T05:30:48,648 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 5206502d6b4766aaa207f7e6fc8fc803 in 26ms, sequenceid=89, compaction requested=false
2024-12-06T05:30:48,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5206502d6b4766aaa207f7e6fc8fc803:

2024-12-06T05:30:48,648 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=81.6 K, sizeToCheck=16.0 K
2024-12-06T05:30:48,648 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:30:48,648 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/d2f83d55717b40aeb2b38101ca4718ae because midkey is the same as first or last row
2024-12-06T05:30:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:48,649 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5206502d6b4766aaa207f7e6fc8fc803 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-06T05:30:48,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/46d21f5ee7a14912883d56a8bede12ff is 1080, key is row0069/info:/1733463048624/Put/seqid=0
2024-12-06T05:30:48,658 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741845_1021 (size=17894)
2024-12-06T05:30:48,658 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741845_1021 (size=17894)
2024-12-06T05:30:48,659 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/46d21f5ee7a14912883d56a8bede12ff
2024-12-06T05:30:48,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/46d21f5ee7a14912883d56a8bede12ff as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/46d21f5ee7a14912883d56a8bede12ff
2024-12-06T05:30:48,672 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/46d21f5ee7a14912883d56a8bede12ff, entries=12, sequenceid=104, filesize=17.5 K
2024-12-06T05:30:48,673 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for 5206502d6b4766aaa207f7e6fc8fc803 in 24ms, sequenceid=104, compaction requested=true
2024-12-06T05:30:48,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5206502d6b4766aaa207f7e6fc8fc803:

2024-12-06T05:30:48,673 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=99.0 K, sizeToCheck=16.0 K
2024-12-06T05:30:48,673 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:30:48,673 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/d2f83d55717b40aeb2b38101ca4718ae because midkey is the same as first or last row
2024-12-06T05:30:48,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5206502d6b4766aaa207f7e6fc8fc803:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T05:30:48,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:30:48,673 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T05:30:48,675 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 101404 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T05:30:48,675 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1541): 5206502d6b4766aaa207f7e6fc8fc803/info is initiating minor compaction (all files)
2024-12-06T05:30:48,675 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5206502d6b4766aaa207f7e6fc8fc803/info in TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.
2024-12-06T05:30:48,675 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/d2f83d55717b40aeb2b38101ca4718ae, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/bdeadd37e6c349c0aa77f7f777f8ea89, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/46d21f5ee7a14912883d56a8bede12ff] into tmpdir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp, totalSize=99.0 K
2024-12-06T05:30:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:48,675 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5206502d6b4766aaa207f7e6fc8fc803 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-06T05:30:48,675 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2f83d55717b40aeb2b38101ca4718ae, keycount=61, bloomtype=ROW, size=69.3 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733463044467
2024-12-06T05:30:48,676 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdeadd37e6c349c0aa77f7f777f8ea89, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733463046604
2024-12-06T05:30:48,676 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46d21f5ee7a14912883d56a8bede12ff, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733463048624
2024-12-06T05:30:48,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/31e93d720cb245e3b7274624b8aa3979 is 1080, key is row0081/info:/1733463048650/Put/seqid=0
2024-12-06T05:30:48,691 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5206502d6b4766aaa207f7e6fc8fc803#info#compaction#65 average throughput is 27.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T05:30:48,692 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/ede1a7be0f7648ed9e26ece8a508dcdc is 1080, key is row0001/info:/1733463044467/Put/seqid=0
2024-12-06T05:30:48,692 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741846_1022 (size=17894)
2024-12-06T05:30:48,693 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741846_1022 (size=17894)
2024-12-06T05:30:48,696 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741847_1023 (size=91639)
2024-12-06T05:30:48,696 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741847_1023 (size=91639)
2024-12-06T05:30:48,703 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/ede1a7be0f7648ed9e26ece8a508dcdc as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/ede1a7be0f7648ed9e26ece8a508dcdc
2024-12-06T05:30:48,709 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5206502d6b4766aaa207f7e6fc8fc803/info of 5206502d6b4766aaa207f7e6fc8fc803 into ede1a7be0f7648ed9e26ece8a508dcdc(size=89.5 K), total size for store is 89.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T05:30:48,709 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5206502d6b4766aaa207f7e6fc8fc803:

2024-12-06T05:30:48,710 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803., storeName=5206502d6b4766aaa207f7e6fc8fc803/info, priority=13, startTime=1733463048673; duration=0sec
2024-12-06T05:30:48,710 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=89.5 K, sizeToCheck=16.0 K
2024-12-06T05:30:48,710 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:30:48,710 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=89.5 K, sizeToCheck=16.0 K
2024-12-06T05:30:48,710 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:30:48,710 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=89.5 K, sizeToCheck=16.0 K
2024-12-06T05:30:48,710 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T05:30:48,711 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:30:48,711 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:30:48,711 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5206502d6b4766aaa207f7e6fc8fc803:info
2024-12-06T05:30:48,713 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44883 {}] assignment.AssignmentManager(1363): Split request from ec46afd49254,42957,1733463033291, parent={ENCODED => 5206502d6b4766aaa207f7e6fc8fc803, NAME => 'TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-12-06T05:30:48,719 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44883 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=ec46afd49254,42957,1733463033291
2024-12-06T05:30:48,724 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44883 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=5206502d6b4766aaa207f7e6fc8fc803, daughterA=b02b1ef6591d9f4c0384547bbc849d53, daughterB=b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:30:48,725 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=5206502d6b4766aaa207f7e6fc8fc803, daughterA=b02b1ef6591d9f4c0384547bbc849d53, daughterB=b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:30:48,725 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=5206502d6b4766aaa207f7e6fc8fc803, daughterA=b02b1ef6591d9f4c0384547bbc849d53, daughterB=b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:30:48,725 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=5206502d6b4766aaa207f7e6fc8fc803, daughterA=b02b1ef6591d9f4c0384547bbc849d53, daughterB=b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:30:48,733 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5206502d6b4766aaa207f7e6fc8fc803, UNASSIGN}]
2024-12-06T05:30:48,735 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5206502d6b4766aaa207f7e6fc8fc803, UNASSIGN
2024-12-06T05:30:48,736 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5206502d6b4766aaa207f7e6fc8fc803, regionState=CLOSING, regionLocation=ec46afd49254,42957,1733463033291
2024-12-06T05:30:48,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5206502d6b4766aaa207f7e6fc8fc803, UNASSIGN because future has completed
2024-12-06T05:30:48,739 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-12-06T05:30:48,739 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5206502d6b4766aaa207f7e6fc8fc803, server=ec46afd49254,42957,1733463033291}]
2024-12-06T05:30:48,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:48,899 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:48,899 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-12-06T05:30:48,900 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing 5206502d6b4766aaa207f7e6fc8fc803, disabling compactions & flushes
2024-12-06T05:30:48,900 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.
2024-12-06T05:30:48,945 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:49,090 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/31e93d720cb245e3b7274624b8aa3979
2024-12-06T05:30:49,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/31e93d720cb245e3b7274624b8aa3979 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/31e93d720cb245e3b7274624b8aa3979
2024-12-06T05:30:49,104 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/31e93d720cb245e3b7274624b8aa3979, entries=12, sequenceid=119, filesize=17.5 K
2024-12-06T05:30:49,105 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=4.20 KB/4304 for 5206502d6b4766aaa207f7e6fc8fc803 in 430ms, sequenceid=119, compaction requested=false
2024-12-06T05:30:49,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5206502d6b4766aaa207f7e6fc8fc803:

2024-12-06T05:30:49,105 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.
2024-12-06T05:30:49,105 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.
2024-12-06T05:30:49,105 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803. after waiting 0 ms
2024-12-06T05:30:49,105 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.
2024-12-06T05:30:49,105 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing 5206502d6b4766aaa207f7e6fc8fc803 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-12-06T05:30:49,109 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/c7fd43bdeb4c4c4a99e75c4f14fa56d3 is 1080, key is row0093/info:/1733463048677/Put/seqid=0
2024-12-06T05:30:49,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741848_1024 (size=9270)
2024-12-06T05:30:49,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741848_1024 (size=9270)
2024-12-06T05:30:49,114 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/c7fd43bdeb4c4c4a99e75c4f14fa56d3
2024-12-06T05:30:49,120 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/.tmp/info/c7fd43bdeb4c4c4a99e75c4f14fa56d3 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/c7fd43bdeb4c4c4a99e75c4f14fa56d3
2024-12-06T05:30:49,126 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/c7fd43bdeb4c4c4a99e75c4f14fa56d3, entries=4, sequenceid=127, filesize=9.1 K
2024-12-06T05:30:49,127 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 5206502d6b4766aaa207f7e6fc8fc803 in 22ms, sequenceid=127, compaction requested=true
2024-12-06T05:30:49,128 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/42801156f20f4a41a6efbfcf128ee6f3, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/3420dcab666541d4bf1908f74a80e8fe, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/f83fb54be9ab49f58d57e064f9e282f2, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/961c8f9bab2946d3b738b924e191a092, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/d8487465af86407ba3a81894427017e9, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/d2f83d55717b40aeb2b38101ca4718ae, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/18bf52f669cb402d8a2ac1cdfb3aba8f, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/bdeadd37e6c349c0aa77f7f777f8ea89, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/46d21f5ee7a14912883d56a8bede12ff] to archive
2024-12-06T05:30:49,129 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-06T05:30:49,131 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/42801156f20f4a41a6efbfcf128ee6f3 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/42801156f20f4a41a6efbfcf128ee6f3
2024-12-06T05:30:49,132 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/3420dcab666541d4bf1908f74a80e8fe to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/3420dcab666541d4bf1908f74a80e8fe
2024-12-06T05:30:49,133 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/f83fb54be9ab49f58d57e064f9e282f2 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/f83fb54be9ab49f58d57e064f9e282f2
2024-12-06T05:30:49,134 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/961c8f9bab2946d3b738b924e191a092 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/961c8f9bab2946d3b738b924e191a092
2024-12-06T05:30:49,135 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/d8487465af86407ba3a81894427017e9 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/d8487465af86407ba3a81894427017e9
2024-12-06T05:30:49,137 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/d2f83d55717b40aeb2b38101ca4718ae to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/d2f83d55717b40aeb2b38101ca4718ae
2024-12-06T05:30:49,138 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/18bf52f669cb402d8a2ac1cdfb3aba8f to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/18bf52f669cb402d8a2ac1cdfb3aba8f
2024-12-06T05:30:49,139 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/bdeadd37e6c349c0aa77f7f777f8ea89 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/bdeadd37e6c349c0aa77f7f777f8ea89
2024-12-06T05:30:49,140 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/46d21f5ee7a14912883d56a8bede12ff to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/46d21f5ee7a14912883d56a8bede12ff
2024-12-06T05:30:49,146 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=1
2024-12-06T05:30:49,146 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.
2024-12-06T05:30:49,147 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for 5206502d6b4766aaa207f7e6fc8fc803:
Waiting for close lock at 1733463048900Running coprocessor pre-close hooks at 1733463048900Disabling compacts and flushes for region at 1733463048900Disabling writes for close at 1733463049105 (+205 ms)Obtaining lock to block concurrent updates at 1733463049105Preparing flush snapshotting stores in 5206502d6b4766aaa207f7e6fc8fc803 at 1733463049105Finished memstore snapshotting TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803., syncing WAL and waiting on mvcc, flushsize=dataSize=4304, getHeapSize=4848, getOffHeapSize=0, getCellsCount=4 at 1733463049105Flushing stores of TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803. at 1733463049106 (+1 ms)Flushing 5206502d6b4766aaa207f7e6fc8fc803/info: creating writer at 1733463049106Flushing 5206502d6b4766aaa207f7e6fc8fc803/info: appending metadata at 1733463049109 (+3 ms)Flushing 5206502d6b4766aaa207f7e6fc8fc803/info: closing flushed file at 1733463049109Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dd4be10: reopening flushed file at 1733463049119 (+10 ms)Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 5206502d6b4766aaa207f7e6fc8fc803 in 22ms, sequenceid=127, compaction requested=true at 1733463049127 (+8 ms)Writing region close event to WAL at 1733463049142 (+15 ms)Running coprocessor post-close hooks at 1733463049146 (+4 ms)Closed at 1733463049146
2024-12-06T05:30:49,149 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:49,149 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5206502d6b4766aaa207f7e6fc8fc803, regionState=CLOSED
2024-12-06T05:30:49,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5206502d6b4766aaa207f7e6fc8fc803, server=ec46afd49254,42957,1733463033291 because future has completed
2024-12-06T05:30:49,154 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-12-06T05:30:49,155 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure 5206502d6b4766aaa207f7e6fc8fc803, server=ec46afd49254,42957,1733463033291 in 413 msec
2024-12-06T05:30:49,156 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-12-06T05:30:49,157 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5206502d6b4766aaa207f7e6fc8fc803, UNASSIGN in 422 msec
2024-12-06T05:30:49,165 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:49,168 INFO  [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=5206502d6b4766aaa207f7e6fc8fc803, threads=3
2024-12-06T05:30:49,170 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/31e93d720cb245e3b7274624b8aa3979 for region: 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:49,170 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/ede1a7be0f7648ed9e26ece8a508dcdc for region: 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:49,170 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/c7fd43bdeb4c4c4a99e75c4f14fa56d3 for region: 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:49,181 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/31e93d720cb245e3b7274624b8aa3979, top=true
2024-12-06T05:30:49,183 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/c7fd43bdeb4c4c4a99e75c4f14fa56d3, top=true
2024-12-06T05:30:49,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741849_1025 (size=27)
2024-12-06T05:30:49,191 INFO  [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/TestLogRolling-testLogRolling=5206502d6b4766aaa207f7e6fc8fc803-c7fd43bdeb4c4c4a99e75c4f14fa56d3 for child: b07d9cd4230ca8093f260ec7e989e136, parent: 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:49,191 INFO  [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/TestLogRolling-testLogRolling=5206502d6b4766aaa207f7e6fc8fc803-31e93d720cb245e3b7274624b8aa3979 for child: b07d9cd4230ca8093f260ec7e989e136, parent: 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:49,191 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/c7fd43bdeb4c4c4a99e75c4f14fa56d3 for region: 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:49,191 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/31e93d720cb245e3b7274624b8aa3979 for region: 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:49,192 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741849_1025 (size=27)
2024-12-06T05:30:49,200 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741850_1026 (size=27)
2024-12-06T05:30:49,201 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741850_1026 (size=27)
2024-12-06T05:30:49,201 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/ede1a7be0f7648ed9e26ece8a508dcdc for region: 5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:30:49,203 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 5206502d6b4766aaa207f7e6fc8fc803 Daughter A: [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b02b1ef6591d9f4c0384547bbc849d53/info/ede1a7be0f7648ed9e26ece8a508dcdc.5206502d6b4766aaa207f7e6fc8fc803] storefiles, Daughter B: [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/TestLogRolling-testLogRolling=5206502d6b4766aaa207f7e6fc8fc803-31e93d720cb245e3b7274624b8aa3979, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/TestLogRolling-testLogRolling=5206502d6b4766aaa207f7e6fc8fc803-c7fd43bdeb4c4c4a99e75c4f14fa56d3, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ede1a7be0f7648ed9e26ece8a508dcdc.5206502d6b4766aaa207f7e6fc8fc803] storefiles.
2024-12-06T05:30:49,211 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741851_1027 (size=71)
2024-12-06T05:30:49,211 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741851_1027 (size=71)
2024-12-06T05:30:49,215 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:49,228 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741852_1028 (size=71)
2024-12-06T05:30:49,229 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741852_1028 (size=71)
2024-12-06T05:30:49,230 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:49,241 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b02b1ef6591d9f4c0384547bbc849d53/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-12-06T05:30:49,244 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-12-06T05:30:49,246 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733463049246"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1733463049246"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1733463049246"}]},"ts":"1733463049246"}
2024-12-06T05:30:49,247 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733463049246"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463049246"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733463049246"}]},"ts":"1733463049246"}
2024-12-06T05:30:49,247 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733463049246"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463049246"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733463049246"}]},"ts":"1733463049246"}
2024-12-06T05:30:49,266 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b02b1ef6591d9f4c0384547bbc849d53, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b07d9cd4230ca8093f260ec7e989e136, ASSIGN}]
2024-12-06T05:30:49,267 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b02b1ef6591d9f4c0384547bbc849d53, ASSIGN
2024-12-06T05:30:49,267 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b07d9cd4230ca8093f260ec7e989e136, ASSIGN
2024-12-06T05:30:49,268 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b02b1ef6591d9f4c0384547bbc849d53, ASSIGN; state=SPLITTING_NEW, location=ec46afd49254,42957,1733463033291; forceNewPlan=false, retain=false
2024-12-06T05:30:49,268 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b07d9cd4230ca8093f260ec7e989e136, ASSIGN; state=SPLITTING_NEW, location=ec46afd49254,42957,1733463033291; forceNewPlan=false, retain=false
2024-12-06T05:30:49,419 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=b07d9cd4230ca8093f260ec7e989e136, regionState=OPENING, regionLocation=ec46afd49254,42957,1733463033291
2024-12-06T05:30:49,419 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=b02b1ef6591d9f4c0384547bbc849d53, regionState=OPENING, regionLocation=ec46afd49254,42957,1733463033291
2024-12-06T05:30:49,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b07d9cd4230ca8093f260ec7e989e136, ASSIGN because future has completed
2024-12-06T05:30:49,421 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure b07d9cd4230ca8093f260ec7e989e136, server=ec46afd49254,42957,1733463033291}]
2024-12-06T05:30:49,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b02b1ef6591d9f4c0384547bbc849d53, ASSIGN because future has completed
2024-12-06T05:30:49,423 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure b02b1ef6591d9f4c0384547bbc849d53, server=ec46afd49254,42957,1733463033291}]
2024-12-06T05:30:49,577 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.
2024-12-06T05:30:49,578 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => b02b1ef6591d9f4c0384547bbc849d53, NAME => 'TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.', STARTKEY => '', ENDKEY => 'row0062'}
2024-12-06T05:30:49,578 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling b02b1ef6591d9f4c0384547bbc849d53
2024-12-06T05:30:49,578 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:30:49,578 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for b02b1ef6591d9f4c0384547bbc849d53
2024-12-06T05:30:49,578 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for b02b1ef6591d9f4c0384547bbc849d53
2024-12-06T05:30:49,579 INFO  [StoreOpener-b02b1ef6591d9f4c0384547bbc849d53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region b02b1ef6591d9f4c0384547bbc849d53 
2024-12-06T05:30:49,580 INFO  [StoreOpener-b02b1ef6591d9f4c0384547bbc849d53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b02b1ef6591d9f4c0384547bbc849d53 columnFamilyName info
2024-12-06T05:30:49,580 DEBUG [StoreOpener-b02b1ef6591d9f4c0384547bbc849d53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:49,593 DEBUG [StoreOpener-b02b1ef6591d9f4c0384547bbc849d53-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b02b1ef6591d9f4c0384547bbc849d53/info/ede1a7be0f7648ed9e26ece8a508dcdc.5206502d6b4766aaa207f7e6fc8fc803->hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/ede1a7be0f7648ed9e26ece8a508dcdc-bottom
2024-12-06T05:30:49,594 INFO  [StoreOpener-b02b1ef6591d9f4c0384547bbc849d53-1 {}] regionserver.HStore(327): Store=b02b1ef6591d9f4c0384547bbc849d53/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:30:49,594 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for b02b1ef6591d9f4c0384547bbc849d53
2024-12-06T05:30:49,595 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b02b1ef6591d9f4c0384547bbc849d53
2024-12-06T05:30:49,596 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b02b1ef6591d9f4c0384547bbc849d53
2024-12-06T05:30:49,596 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for b02b1ef6591d9f4c0384547bbc849d53
2024-12-06T05:30:49,596 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for b02b1ef6591d9f4c0384547bbc849d53
2024-12-06T05:30:49,598 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for b02b1ef6591d9f4c0384547bbc849d53
2024-12-06T05:30:49,599 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened b02b1ef6591d9f4c0384547bbc849d53; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=717291, jitterRate=-0.08791816234588623}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-06T05:30:49,599 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b02b1ef6591d9f4c0384547bbc849d53
2024-12-06T05:30:49,599 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for b02b1ef6591d9f4c0384547bbc849d53:
Running coprocessor pre-open hook at 1733463049578Writing region info on filesystem at 1733463049578Initializing all the Stores at 1733463049579 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463049579Cleaning up temporary data from old regions at 1733463049596 (+17 ms)Running coprocessor post-open hooks at 1733463049599 (+3 ms)Region opened successfully at 1733463049599
2024-12-06T05:30:49,600 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53., pid=13, masterSystemTime=1733463049574
2024-12-06T05:30:49,600 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store b02b1ef6591d9f4c0384547bbc849d53:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T05:30:49,600 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:30:49,600 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-12-06T05:30:49,601 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.
2024-12-06T05:30:49,601 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1541): b02b1ef6591d9f4c0384547bbc849d53/info is initiating minor compaction (all files)
2024-12-06T05:30:49,601 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b02b1ef6591d9f4c0384547bbc849d53/info in TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.
2024-12-06T05:30:49,602 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b02b1ef6591d9f4c0384547bbc849d53/info/ede1a7be0f7648ed9e26ece8a508dcdc.5206502d6b4766aaa207f7e6fc8fc803->hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/ede1a7be0f7648ed9e26ece8a508dcdc-bottom] into tmpdir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b02b1ef6591d9f4c0384547bbc849d53/.tmp, totalSize=89.5 K
2024-12-06T05:30:49,602 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ede1a7be0f7648ed9e26ece8a508dcdc.5206502d6b4766aaa207f7e6fc8fc803, keycount=40, bloomtype=ROW, size=89.5 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733463044467
2024-12-06T05:30:49,603 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.
2024-12-06T05:30:49,603 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.
2024-12-06T05:30:49,603 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.
2024-12-06T05:30:49,603 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => b07d9cd4230ca8093f260ec7e989e136, NAME => 'TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.', STARTKEY => 'row0062', ENDKEY => ''}
2024-12-06T05:30:49,603 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:30:49,603 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:30:49,604 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:30:49,604 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=b02b1ef6591d9f4c0384547bbc849d53, regionState=OPEN, openSeqNum=131, regionLocation=ec46afd49254,42957,1733463033291
2024-12-06T05:30:49,604 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:30:49,605 INFO  [StoreOpener-b07d9cd4230ca8093f260ec7e989e136-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region b07d9cd4230ca8093f260ec7e989e136 
2024-12-06T05:30:49,606 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-12-06T05:30:49,606 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-12-06T05:30:49,606 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB
2024-12-06T05:30:49,606 INFO  [StoreOpener-b07d9cd4230ca8093f260ec7e989e136-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b07d9cd4230ca8093f260ec7e989e136 columnFamilyName info
2024-12-06T05:30:49,606 DEBUG [StoreOpener-b07d9cd4230ca8093f260ec7e989e136-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:30:49,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure b02b1ef6591d9f4c0384547bbc849d53, server=ec46afd49254,42957,1733463033291 because future has completed
2024-12-06T05:30:49,610 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10
2024-12-06T05:30:49,610 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure b02b1ef6591d9f4c0384547bbc849d53, server=ec46afd49254,42957,1733463033291 in 185 msec
2024-12-06T05:30:49,612 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b02b1ef6591d9f4c0384547bbc849d53, ASSIGN in 344 msec
2024-12-06T05:30:49,625 DEBUG [StoreOpener-b07d9cd4230ca8093f260ec7e989e136-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/TestLogRolling-testLogRolling=5206502d6b4766aaa207f7e6fc8fc803-31e93d720cb245e3b7274624b8aa3979
2024-12-06T05:30:49,626 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b02b1ef6591d9f4c0384547bbc849d53#info#compaction#67 average throughput is 15.65 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T05:30:49,627 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b02b1ef6591d9f4c0384547bbc849d53/.tmp/info/c00522a46dab44feb7eeb0dc9f0da17d is 1080, key is row0001/info:/1733463044467/Put/seqid=0
2024-12-06T05:30:49,631 DEBUG [StoreOpener-b07d9cd4230ca8093f260ec7e989e136-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/TestLogRolling-testLogRolling=5206502d6b4766aaa207f7e6fc8fc803-c7fd43bdeb4c4c4a99e75c4f14fa56d3
2024-12-06T05:30:49,637 DEBUG [StoreOpener-b07d9cd4230ca8093f260ec7e989e136-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ede1a7be0f7648ed9e26ece8a508dcdc.5206502d6b4766aaa207f7e6fc8fc803->hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/ede1a7be0f7648ed9e26ece8a508dcdc-top
2024-12-06T05:30:49,637 INFO  [StoreOpener-b07d9cd4230ca8093f260ec7e989e136-1 {}] regionserver.HStore(327): Store=b07d9cd4230ca8093f260ec7e989e136/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:30:49,637 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:30:49,638 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:30:49,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/.tmp/info/589567979d3747eb98979c83534bec21 is 193, key is TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136./info:regioninfo/1733463049419/Put/seqid=0
2024-12-06T05:30:49,639 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:30:49,640 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:30:49,640 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:30:49,642 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741853_1029 (size=70862)
2024-12-06T05:30:49,642 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741853_1029 (size=70862)
2024-12-06T05:30:49,643 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:30:49,644 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened b07d9cd4230ca8093f260ec7e989e136; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=832707, jitterRate=0.058842360973358154}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-06T05:30:49,644 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:30:49,644 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for b07d9cd4230ca8093f260ec7e989e136:
Running coprocessor pre-open hook at 1733463049604Writing region info on filesystem at 1733463049604Initializing all the Stores at 1733463049604Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463049604Cleaning up temporary data from old regions at 1733463049640 (+36 ms)Running coprocessor post-open hooks at 1733463049644 (+4 ms)Region opened successfully at 1733463049644
2024-12-06T05:30:49,645 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136., pid=12, masterSystemTime=1733463049574
2024-12-06T05:30:49,645 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store b07d9cd4230ca8093f260ec7e989e136:info, priority=-2147483648, current under compaction store size is 2
2024-12-06T05:30:49,645 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741854_1030 (size=9847)
2024-12-06T05:30:49,645 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0
2024-12-06T05:30:49,646 DEBUG [RS:0;ec46afd49254:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T05:30:49,646 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741854_1030 (size=9847)
2024-12-06T05:30:49,647 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/.tmp/info/589567979d3747eb98979c83534bec21
2024-12-06T05:30:49,648 INFO  [RS:0;ec46afd49254:42957-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.
2024-12-06T05:30:49,648 DEBUG [RS:0;ec46afd49254:42957-longCompactions-0 {}] regionserver.HStore(1541): b07d9cd4230ca8093f260ec7e989e136/info is initiating minor compaction (all files)
2024-12-06T05:30:49,648 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.
2024-12-06T05:30:49,648 INFO  [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.
2024-12-06T05:30:49,648 INFO  [RS:0;ec46afd49254:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b07d9cd4230ca8093f260ec7e989e136/info in TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.
2024-12-06T05:30:49,648 INFO  [RS:0;ec46afd49254:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ede1a7be0f7648ed9e26ece8a508dcdc.5206502d6b4766aaa207f7e6fc8fc803->hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/ede1a7be0f7648ed9e26ece8a508dcdc-top, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/TestLogRolling-testLogRolling=5206502d6b4766aaa207f7e6fc8fc803-31e93d720cb245e3b7274624b8aa3979, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/TestLogRolling-testLogRolling=5206502d6b4766aaa207f7e6fc8fc803-c7fd43bdeb4c4c4a99e75c4f14fa56d3] into tmpdir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp, totalSize=116.0 K
2024-12-06T05:30:49,649 DEBUG [RS:0;ec46afd49254:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting ede1a7be0f7648ed9e26ece8a508dcdc.5206502d6b4766aaa207f7e6fc8fc803, keycount=40, bloomtype=ROW, size=89.5 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733463044467
2024-12-06T05:30:49,649 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=b07d9cd4230ca8093f260ec7e989e136, regionState=OPEN, openSeqNum=131, regionLocation=ec46afd49254,42957,1733463033291
2024-12-06T05:30:49,650 DEBUG [RS:0;ec46afd49254:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=5206502d6b4766aaa207f7e6fc8fc803-31e93d720cb245e3b7274624b8aa3979, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733463048650
2024-12-06T05:30:49,651 DEBUG [RS:0;ec46afd49254:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=5206502d6b4766aaa207f7e6fc8fc803-c7fd43bdeb4c4c4a99e75c4f14fa56d3, keycount=4, bloomtype=ROW, size=9.1 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733463048677
2024-12-06T05:30:49,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure b07d9cd4230ca8093f260ec7e989e136, server=ec46afd49254,42957,1733463033291 because future has completed
2024-12-06T05:30:49,652 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b02b1ef6591d9f4c0384547bbc849d53/.tmp/info/c00522a46dab44feb7eeb0dc9f0da17d as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b02b1ef6591d9f4c0384547bbc849d53/info/c00522a46dab44feb7eeb0dc9f0da17d
2024-12-06T05:30:49,657 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-12-06T05:30:49,657 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure b07d9cd4230ca8093f260ec7e989e136, server=ec46afd49254,42957,1733463033291 in 232 msec
2024-12-06T05:30:49,661 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in b02b1ef6591d9f4c0384547bbc849d53/info of b02b1ef6591d9f4c0384547bbc849d53 into c00522a46dab44feb7eeb0dc9f0da17d(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T05:30:49,661 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b02b1ef6591d9f4c0384547bbc849d53:

2024-12-06T05:30:49,661 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53., storeName=b02b1ef6591d9f4c0384547bbc849d53/info, priority=15, startTime=1733463049600; duration=0sec
2024-12-06T05:30:49,661 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:30:49,661 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b02b1ef6591d9f4c0384547bbc849d53:info
2024-12-06T05:30:49,662 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7
2024-12-06T05:30:49,662 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b07d9cd4230ca8093f260ec7e989e136, ASSIGN in 391 msec
2024-12-06T05:30:49,665 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=5206502d6b4766aaa207f7e6fc8fc803, daughterA=b02b1ef6591d9f4c0384547bbc849d53, daughterB=b07d9cd4230ca8093f260ec7e989e136 in 943 msec
2024-12-06T05:30:49,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/.tmp/ns/6c7b340b258c48cdb3dabc4699c3508b is 43, key is default/ns:d/1733463034351/Put/seqid=0
2024-12-06T05:30:49,676 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741855_1031 (size=5153)
2024-12-06T05:30:49,676 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741855_1031 (size=5153)
2024-12-06T05:30:49,677 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/.tmp/ns/6c7b340b258c48cdb3dabc4699c3508b
2024-12-06T05:30:49,682 INFO  [RS:0;ec46afd49254:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b07d9cd4230ca8093f260ec7e989e136#info#compaction#70 average throughput is 35.92 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T05:30:49,683 DEBUG [RS:0;ec46afd49254:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/ff82c6853ee647ec9e8edaf264164b25 is 1080, key is row0062/info:/1733463046604/Put/seqid=0
2024-12-06T05:30:49,694 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741856_1032 (size=42984)
2024-12-06T05:30:49,694 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741856_1032 (size=42984)
2024-12-06T05:30:49,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/.tmp/table/e1e458d89c9b488e8de561dd0adb99a4 is 65, key is TestLogRolling-testLogRolling/table:state/1733463034797/Put/seqid=0
2024-12-06T05:30:49,700 DEBUG [RS:0;ec46afd49254:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/ff82c6853ee647ec9e8edaf264164b25 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ff82c6853ee647ec9e8edaf264164b25
2024-12-06T05:30:49,705 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741857_1033 (size=5340)
2024-12-06T05:30:49,707 INFO  [RS:0;ec46afd49254:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b07d9cd4230ca8093f260ec7e989e136/info of b07d9cd4230ca8093f260ec7e989e136 into ff82c6853ee647ec9e8edaf264164b25(size=42.0 K), total size for store is 42.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T05:30:49,707 DEBUG [RS:0;ec46afd49254:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:30:49,707 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741857_1033 (size=5340)
2024-12-06T05:30:49,707 INFO  [RS:0;ec46afd49254:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136., storeName=b07d9cd4230ca8093f260ec7e989e136/info, priority=13, startTime=1733463049645; duration=0sec
2024-12-06T05:30:49,707 DEBUG [RS:0;ec46afd49254:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:30:49,707 DEBUG [RS:0;ec46afd49254:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b07d9cd4230ca8093f260ec7e989e136:info
2024-12-06T05:30:49,708 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/.tmp/table/e1e458d89c9b488e8de561dd0adb99a4
2024-12-06T05:30:49,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/.tmp/info/589567979d3747eb98979c83534bec21 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/info/589567979d3747eb98979c83534bec21
2024-12-06T05:30:49,721 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/info/589567979d3747eb98979c83534bec21, entries=30, sequenceid=17, filesize=9.6 K
2024-12-06T05:30:49,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/.tmp/ns/6c7b340b258c48cdb3dabc4699c3508b as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/ns/6c7b340b258c48cdb3dabc4699c3508b
2024-12-06T05:30:49,727 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/ns/6c7b340b258c48cdb3dabc4699c3508b, entries=2, sequenceid=17, filesize=5.0 K
2024-12-06T05:30:49,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/.tmp/table/e1e458d89c9b488e8de561dd0adb99a4 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/table/e1e458d89c9b488e8de561dd0adb99a4
2024-12-06T05:30:49,735 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/table/e1e458d89c9b488e8de561dd0adb99a4, entries=2, sequenceid=17, filesize=5.2 K
2024-12-06T05:30:49,736 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 130ms, sequenceid=17, compaction requested=false
2024-12-06T05:30:49,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-06T05:30:49,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:49,946 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.3:58284 deadline: 1733463060688, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803. is not online on ec46afd49254,42957,1733463033291
2024-12-06T05:30:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803., hostname=ec46afd49254,42957,1733463033291, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803., hostname=ec46afd49254,42957,1733463033291, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803. is not online on ec46afd49254,42957,1733463033291
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-06T05:30:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803., hostname=ec46afd49254,42957,1733463033291, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803. is not online on ec46afd49254,42957,1733463033291
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-06T05:30:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1733463034429.5206502d6b4766aaa207f7e6fc8fc803., hostname=ec46afd49254,42957,1733463033291, seqNum=2 from cache
2024-12-06T05:30:50,771 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:50,946 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:51,772 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:51,947 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:52,772 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:52,947 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:53,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:53,948 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:54,147 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,147 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,147 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,147 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,148 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,148 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,148 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,149 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,169 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,170 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,170 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,170 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,170 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,170 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,173 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,173 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,173 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,175 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,683 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T05:30:54,684 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,684 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,684 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,684 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,685 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,685 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,686 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,686 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,708 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,708 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,708 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,708 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,708 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,709 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,712 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,712 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,713 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,715 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:30:54,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:54,949 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:55,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:55,949 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:56,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:56,950 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:57,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:57,950 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:58,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:58,951 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:59,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:30:59,952 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:00,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136., hostname=ec46afd49254,42957,1733463033291, seqNum=131]
2024-12-06T05:31:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:31:00,797 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b07d9cd4230ca8093f260ec7e989e136 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T05:31:00,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/d3e4d2aa9ea444afb7591242d7a4e19e is 1080, key is row0097/info:/1733463060785/Put/seqid=0
2024-12-06T05:31:00,811 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741858_1034 (size=12516)
2024-12-06T05:31:00,811 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741858_1034 (size=12516)
2024-12-06T05:31:00,812 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/d3e4d2aa9ea444afb7591242d7a4e19e
2024-12-06T05:31:00,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/d3e4d2aa9ea444afb7591242d7a4e19e as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/d3e4d2aa9ea444afb7591242d7a4e19e
2024-12-06T05:31:00,827 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/d3e4d2aa9ea444afb7591242d7a4e19e, entries=7, sequenceid=141, filesize=12.2 K
2024-12-06T05:31:00,828 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for b07d9cd4230ca8093f260ec7e989e136 in 31ms, sequenceid=141, compaction requested=false
2024-12-06T05:31:00,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:31:00,830 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b07d9cd4230ca8093f260ec7e989e136 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-12-06T05:31:00,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/9a4b1754875f41c7a359c8ae94ba1845 is 1080, key is row0104/info:/1733463060798/Put/seqid=0
2024-12-06T05:31:00,845 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741859_1035 (size=21156)
2024-12-06T05:31:00,846 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741859_1035 (size=21156)
2024-12-06T05:31:00,846 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/9a4b1754875f41c7a359c8ae94ba1845
2024-12-06T05:31:00,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/9a4b1754875f41c7a359c8ae94ba1845 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/9a4b1754875f41c7a359c8ae94ba1845
2024-12-06T05:31:00,862 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/9a4b1754875f41c7a359c8ae94ba1845, entries=15, sequenceid=159, filesize=20.7 K
2024-12-06T05:31:00,863 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=10.51 KB/10760 for b07d9cd4230ca8093f260ec7e989e136 in 33ms, sequenceid=159, compaction requested=true
2024-12-06T05:31:00,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:00,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b07d9cd4230ca8093f260ec7e989e136:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T05:31:00,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:31:00,864 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T05:31:00,865 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 76656 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T05:31:00,865 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1541): b07d9cd4230ca8093f260ec7e989e136/info is initiating minor compaction (all files)
2024-12-06T05:31:00,865 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b07d9cd4230ca8093f260ec7e989e136/info in TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.
2024-12-06T05:31:00,865 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ff82c6853ee647ec9e8edaf264164b25, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/d3e4d2aa9ea444afb7591242d7a4e19e, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/9a4b1754875f41c7a359c8ae94ba1845] into tmpdir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp, totalSize=74.9 K
2024-12-06T05:31:00,865 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff82c6853ee647ec9e8edaf264164b25, keycount=35, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733463046604
2024-12-06T05:31:00,866 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3e4d2aa9ea444afb7591242d7a4e19e, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733463060785
2024-12-06T05:31:00,866 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a4b1754875f41c7a359c8ae94ba1845, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733463060798
2024-12-06T05:31:00,878 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b07d9cd4230ca8093f260ec7e989e136#info#compaction#74 average throughput is 58.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T05:31:00,878 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/b8b752f870944dd994c604a8ce6d31d6 is 1080, key is row0062/info:/1733463046604/Put/seqid=0
2024-12-06T05:31:00,883 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741860_1036 (size=66870)
2024-12-06T05:31:00,883 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741860_1036 (size=66870)
2024-12-06T05:31:00,888 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/b8b752f870944dd994c604a8ce6d31d6 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/b8b752f870944dd994c604a8ce6d31d6
2024-12-06T05:31:00,894 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b07d9cd4230ca8093f260ec7e989e136/info of b07d9cd4230ca8093f260ec7e989e136 into b8b752f870944dd994c604a8ce6d31d6(size=65.3 K), total size for store is 65.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T05:31:00,894 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:00,894 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136., storeName=b07d9cd4230ca8093f260ec7e989e136/info, priority=13, startTime=1733463060864; duration=0sec
2024-12-06T05:31:00,894 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:31:00,894 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b07d9cd4230ca8093f260ec7e989e136:info
2024-12-06T05:31:00,953 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:01,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:01,953 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:02,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:31:02,853 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b07d9cd4230ca8093f260ec7e989e136 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-12-06T05:31:02,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/51b242d83d014b8a9a00e86af0e23531 is 1080, key is row0119/info:/1733463060831/Put/seqid=0
2024-12-06T05:31:02,864 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741861_1037 (size=16828)
2024-12-06T05:31:02,865 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741861_1037 (size=16828)
2024-12-06T05:31:02,865 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/51b242d83d014b8a9a00e86af0e23531
2024-12-06T05:31:02,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/51b242d83d014b8a9a00e86af0e23531 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/51b242d83d014b8a9a00e86af0e23531
2024-12-06T05:31:02,878 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/51b242d83d014b8a9a00e86af0e23531, entries=11, sequenceid=174, filesize=16.4 K
2024-12-06T05:31:02,879 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=11.56 KB/11836 for b07d9cd4230ca8093f260ec7e989e136 in 26ms, sequenceid=174, compaction requested=false
2024-12-06T05:31:02,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:31:02,881 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b07d9cd4230ca8093f260ec7e989e136 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-06T05:31:02,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/faaded2782ef4a86a2dc27fe05868e3e is 1080, key is row0130/info:/1733463062855/Put/seqid=0
2024-12-06T05:31:02,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741862_1038 (size=17906)
2024-12-06T05:31:02,891 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741862_1038 (size=17906)
2024-12-06T05:31:02,891 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/faaded2782ef4a86a2dc27fe05868e3e
2024-12-06T05:31:02,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/faaded2782ef4a86a2dc27fe05868e3e as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/faaded2782ef4a86a2dc27fe05868e3e
2024-12-06T05:31:02,904 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/faaded2782ef4a86a2dc27fe05868e3e, entries=12, sequenceid=189, filesize=17.5 K
2024-12-06T05:31:02,905 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=11.56 KB/11836 for b07d9cd4230ca8093f260ec7e989e136 in 24ms, sequenceid=189, compaction requested=true
2024-12-06T05:31:02,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:02,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b07d9cd4230ca8093f260ec7e989e136:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T05:31:02,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:31:02,905 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T05:31:02,906 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 101604 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T05:31:02,906 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1541): b07d9cd4230ca8093f260ec7e989e136/info is initiating minor compaction (all files)
2024-12-06T05:31:02,906 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b07d9cd4230ca8093f260ec7e989e136/info in TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.
2024-12-06T05:31:02,906 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/b8b752f870944dd994c604a8ce6d31d6, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/51b242d83d014b8a9a00e86af0e23531, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/faaded2782ef4a86a2dc27fe05868e3e] into tmpdir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp, totalSize=99.2 K
2024-12-06T05:31:02,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:31:02,907 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8b752f870944dd994c604a8ce6d31d6, keycount=57, bloomtype=ROW, size=65.3 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733463046604
2024-12-06T05:31:02,907 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b07d9cd4230ca8093f260ec7e989e136 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-06T05:31:02,907 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51b242d83d014b8a9a00e86af0e23531, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733463060831
2024-12-06T05:31:02,907 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting faaded2782ef4a86a2dc27fe05868e3e, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733463062855
2024-12-06T05:31:02,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/4236d541fa1e4a2ba5111a09e45d04eb is 1080, key is row0142/info:/1733463062882/Put/seqid=0
2024-12-06T05:31:02,926 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741863_1039 (size=17906)
2024-12-06T05:31:02,927 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741863_1039 (size=17906)
2024-12-06T05:31:02,927 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/4236d541fa1e4a2ba5111a09e45d04eb
2024-12-06T05:31:02,928 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b07d9cd4230ca8093f260ec7e989e136#info#compaction#78 average throughput is 27.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T05:31:02,929 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/c262906748104c61a0cc6f715625b53d is 1080, key is row0062/info:/1733463046604/Put/seqid=0
2024-12-06T05:31:02,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/4236d541fa1e4a2ba5111a09e45d04eb as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/4236d541fa1e4a2ba5111a09e45d04eb
2024-12-06T05:31:02,937 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741864_1040 (size=91843)
2024-12-06T05:31:02,937 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741864_1040 (size=91843)
2024-12-06T05:31:02,941 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/4236d541fa1e4a2ba5111a09e45d04eb, entries=12, sequenceid=204, filesize=17.5 K
2024-12-06T05:31:02,942 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=7.36 KB/7532 for b07d9cd4230ca8093f260ec7e989e136 in 36ms, sequenceid=204, compaction requested=false
2024-12-06T05:31:02,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:02,944 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/c262906748104c61a0cc6f715625b53d as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/c262906748104c61a0cc6f715625b53d
2024-12-06T05:31:02,950 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b07d9cd4230ca8093f260ec7e989e136/info of b07d9cd4230ca8093f260ec7e989e136 into c262906748104c61a0cc6f715625b53d(size=89.7 K), total size for store is 107.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T05:31:02,950 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:02,950 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136., storeName=b07d9cd4230ca8093f260ec7e989e136/info, priority=13, startTime=1733463062905; duration=0sec
2024-12-06T05:31:02,950 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:31:02,950 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b07d9cd4230ca8093f260ec7e989e136:info
2024-12-06T05:31:02,954 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:03,093 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T05:31:03,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:03,955 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:04,780 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:04,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:31:04,931 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b07d9cd4230ca8093f260ec7e989e136 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-12-06T05:31:04,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/ff726c570bf84b2783edec2e8bb71708 is 1080, key is row0154/info:/1733463062908/Put/seqid=0
2024-12-06T05:31:04,940 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741865_1041 (size=13594)
2024-12-06T05:31:04,940 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741865_1041 (size=13594)
2024-12-06T05:31:04,940 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/ff726c570bf84b2783edec2e8bb71708
2024-12-06T05:31:04,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/ff726c570bf84b2783edec2e8bb71708 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ff726c570bf84b2783edec2e8bb71708
2024-12-06T05:31:04,951 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ff726c570bf84b2783edec2e8bb71708, entries=8, sequenceid=216, filesize=13.3 K
2024-12-06T05:31:04,952 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=10.51 KB/10760 for b07d9cd4230ca8093f260ec7e989e136 in 21ms, sequenceid=216, compaction requested=true
2024-12-06T05:31:04,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:04,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b07d9cd4230ca8093f260ec7e989e136:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T05:31:04,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:31:04,953 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T05:31:04,954 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 123343 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T05:31:04,954 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1541): b07d9cd4230ca8093f260ec7e989e136/info is initiating minor compaction (all files)
2024-12-06T05:31:04,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:31:04,954 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b07d9cd4230ca8093f260ec7e989e136/info in TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.
2024-12-06T05:31:04,954 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b07d9cd4230ca8093f260ec7e989e136 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-06T05:31:04,954 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/c262906748104c61a0cc6f715625b53d, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/4236d541fa1e4a2ba5111a09e45d04eb, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ff726c570bf84b2783edec2e8bb71708] into tmpdir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp, totalSize=120.5 K
2024-12-06T05:31:04,955 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting c262906748104c61a0cc6f715625b53d, keycount=80, bloomtype=ROW, size=89.7 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733463046604
2024-12-06T05:31:04,955 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4236d541fa1e4a2ba5111a09e45d04eb, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733463062882
2024-12-06T05:31:04,955 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:04,956 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff726c570bf84b2783edec2e8bb71708, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733463062908
2024-12-06T05:31:04,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/ff7de05de208462a84d20e4678bcb65b is 1080, key is row0162/info:/1733463064932/Put/seqid=0
2024-12-06T05:31:04,964 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741866_1042 (size=17906)
2024-12-06T05:31:04,965 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741866_1042 (size=17906)
2024-12-06T05:31:04,965 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/ff7de05de208462a84d20e4678bcb65b
2024-12-06T05:31:04,970 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b07d9cd4230ca8093f260ec7e989e136#info#compaction#81 average throughput is 51.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T05:31:04,971 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/3ef6cc81732846c1991c6cba2cb6185f is 1080, key is row0062/info:/1733463046604/Put/seqid=0
2024-12-06T05:31:04,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/ff7de05de208462a84d20e4678bcb65b as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ff7de05de208462a84d20e4678bcb65b
2024-12-06T05:31:04,977 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741867_1043 (size=113509)
2024-12-06T05:31:04,977 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741867_1043 (size=113509)
2024-12-06T05:31:04,981 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ff7de05de208462a84d20e4678bcb65b, entries=12, sequenceid=231, filesize=17.5 K
2024-12-06T05:31:04,982 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=12.61 KB/12912 for b07d9cd4230ca8093f260ec7e989e136 in 28ms, sequenceid=231, compaction requested=false
2024-12-06T05:31:04,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:31:04,983 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b07d9cd4230ca8093f260ec7e989e136 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-12-06T05:31:04,984 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/3ef6cc81732846c1991c6cba2cb6185f as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/3ef6cc81732846c1991c6cba2cb6185f
2024-12-06T05:31:04,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/f4fbd942eca342409f75de7301fb2351 is 1080, key is row0174/info:/1733463064955/Put/seqid=0
2024-12-06T05:31:04,991 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b07d9cd4230ca8093f260ec7e989e136/info of b07d9cd4230ca8093f260ec7e989e136 into 3ef6cc81732846c1991c6cba2cb6185f(size=110.8 K), total size for store is 128.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T05:31:04,991 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:04,991 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136., storeName=b07d9cd4230ca8093f260ec7e989e136/info, priority=13, startTime=1733463064952; duration=0sec
2024-12-06T05:31:04,991 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:31:04,991 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b07d9cd4230ca8093f260ec7e989e136:info
2024-12-06T05:31:04,992 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741868_1044 (size=19000)
2024-12-06T05:31:04,992 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741868_1044 (size=19000)
2024-12-06T05:31:04,993 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/f4fbd942eca342409f75de7301fb2351
2024-12-06T05:31:04,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/f4fbd942eca342409f75de7301fb2351 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/f4fbd942eca342409f75de7301fb2351
2024-12-06T05:31:05,003 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/f4fbd942eca342409f75de7301fb2351, entries=13, sequenceid=247, filesize=18.6 K
2024-12-06T05:31:05,004 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=6.30 KB/6456 for b07d9cd4230ca8093f260ec7e989e136 in 21ms, sequenceid=247, compaction requested=true
2024-12-06T05:31:05,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:05,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b07d9cd4230ca8093f260ec7e989e136:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T05:31:05,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:31:05,004 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T05:31:05,005 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 150415 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T05:31:05,005 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1541): b07d9cd4230ca8093f260ec7e989e136/info is initiating minor compaction (all files)
2024-12-06T05:31:05,005 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b07d9cd4230ca8093f260ec7e989e136/info in TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.
2024-12-06T05:31:05,006 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/3ef6cc81732846c1991c6cba2cb6185f, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ff7de05de208462a84d20e4678bcb65b, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/f4fbd942eca342409f75de7301fb2351] into tmpdir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp, totalSize=146.9 K
2024-12-06T05:31:05,006 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ef6cc81732846c1991c6cba2cb6185f, keycount=100, bloomtype=ROW, size=110.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733463046604
2024-12-06T05:31:05,006 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff7de05de208462a84d20e4678bcb65b, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733463064932
2024-12-06T05:31:05,006 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4fbd942eca342409f75de7301fb2351, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733463064955
2024-12-06T05:31:05,016 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b07d9cd4230ca8093f260ec7e989e136#info#compaction#83 average throughput is 64.13 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T05:31:05,017 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/38a7bfa312db49da8f1dc8da82976879 is 1080, key is row0062/info:/1733463046604/Put/seqid=0
2024-12-06T05:31:05,021 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741869_1045 (size=140766)
2024-12-06T05:31:05,021 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741869_1045 (size=140766)
2024-12-06T05:31:05,026 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/38a7bfa312db49da8f1dc8da82976879 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/38a7bfa312db49da8f1dc8da82976879
2024-12-06T05:31:05,033 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b07d9cd4230ca8093f260ec7e989e136/info of b07d9cd4230ca8093f260ec7e989e136 into 38a7bfa312db49da8f1dc8da82976879(size=137.5 K), total size for store is 137.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T05:31:05,033 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:05,033 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136., storeName=b07d9cd4230ca8093f260ec7e989e136/info, priority=13, startTime=1733463065004; duration=0sec
2024-12-06T05:31:05,033 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:31:05,033 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b07d9cd4230ca8093f260ec7e989e136:info
2024-12-06T05:31:05,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:05,956 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:06,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:06,956 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:31:06,999 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b07d9cd4230ca8093f260ec7e989e136 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T05:31:07,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/3a043bb4e8c84f108308e09e4ecb9778 is 1080, key is row0187/info:/1733463064985/Put/seqid=0
2024-12-06T05:31:07,015 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741870_1046 (size=12517)
2024-12-06T05:31:07,015 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741870_1046 (size=12517)
2024-12-06T05:31:07,016 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/3a043bb4e8c84f108308e09e4ecb9778
2024-12-06T05:31:07,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/3a043bb4e8c84f108308e09e4ecb9778 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/3a043bb4e8c84f108308e09e4ecb9778
2024-12-06T05:31:07,027 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/3a043bb4e8c84f108308e09e4ecb9778, entries=7, sequenceid=259, filesize=12.2 K
2024-12-06T05:31:07,028 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for b07d9cd4230ca8093f260ec7e989e136 in 29ms, sequenceid=259, compaction requested=false
2024-12-06T05:31:07,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:31:07,030 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b07d9cd4230ca8093f260ec7e989e136 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-06T05:31:07,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/47364b61a3f04c14ac719fa1f36411b6 is 1080, key is row0194/info:/1733463067001/Put/seqid=0
2024-12-06T05:31:07,039 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741871_1047 (size=20092)
2024-12-06T05:31:07,040 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741871_1047 (size=20092)
2024-12-06T05:31:07,042 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/47364b61a3f04c14ac719fa1f36411b6
2024-12-06T05:31:07,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/47364b61a3f04c14ac719fa1f36411b6 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/47364b61a3f04c14ac719fa1f36411b6
2024-12-06T05:31:07,054 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/47364b61a3f04c14ac719fa1f36411b6, entries=14, sequenceid=276, filesize=19.6 K
2024-12-06T05:31:07,055 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=11.56 KB/11836 for b07d9cd4230ca8093f260ec7e989e136 in 26ms, sequenceid=276, compaction requested=true
2024-12-06T05:31:07,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:07,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b07d9cd4230ca8093f260ec7e989e136:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T05:31:07,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:31:07,056 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T05:31:07,057 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 173375 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T05:31:07,057 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1541): b07d9cd4230ca8093f260ec7e989e136/info is initiating minor compaction (all files)
2024-12-06T05:31:07,057 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b07d9cd4230ca8093f260ec7e989e136/info in TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.
2024-12-06T05:31:07,057 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/38a7bfa312db49da8f1dc8da82976879, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/3a043bb4e8c84f108308e09e4ecb9778, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/47364b61a3f04c14ac719fa1f36411b6] into tmpdir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp, totalSize=169.3 K
2024-12-06T05:31:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:31:07,057 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b07d9cd4230ca8093f260ec7e989e136 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-12-06T05:31:07,058 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38a7bfa312db49da8f1dc8da82976879, keycount=125, bloomtype=ROW, size=137.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733463046604
2024-12-06T05:31:07,058 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a043bb4e8c84f108308e09e4ecb9778, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733463064985
2024-12-06T05:31:07,058 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47364b61a3f04c14ac719fa1f36411b6, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733463067001
2024-12-06T05:31:07,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/81c703cf89d547b6a6ceed3123401fe5 is 1080, key is row0208/info:/1733463067031/Put/seqid=0
2024-12-06T05:31:07,073 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741872_1048 (size=19013)
2024-12-06T05:31:07,074 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741872_1048 (size=19013)
2024-12-06T05:31:07,074 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/81c703cf89d547b6a6ceed3123401fe5
2024-12-06T05:31:07,075 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b07d9cd4230ca8093f260ec7e989e136#info#compaction#87 average throughput is 74.91 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T05:31:07,076 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/1db50090dbda45208968667793dfb5da is 1080, key is row0062/info:/1733463046604/Put/seqid=0
2024-12-06T05:31:07,081 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741873_1049 (size=163525)
2024-12-06T05:31:07,082 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741873_1049 (size=163525)
2024-12-06T05:31:07,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/81c703cf89d547b6a6ceed3123401fe5 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/81c703cf89d547b6a6ceed3123401fe5
2024-12-06T05:31:07,087 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/1db50090dbda45208968667793dfb5da as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/1db50090dbda45208968667793dfb5da
2024-12-06T05:31:07,087 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/81c703cf89d547b6a6ceed3123401fe5, entries=13, sequenceid=292, filesize=18.6 K
2024-12-06T05:31:07,088 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=4.20 KB/4304 for b07d9cd4230ca8093f260ec7e989e136 in 31ms, sequenceid=292, compaction requested=false
2024-12-06T05:31:07,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:07,093 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b07d9cd4230ca8093f260ec7e989e136/info of b07d9cd4230ca8093f260ec7e989e136 into 1db50090dbda45208968667793dfb5da(size=159.7 K), total size for store is 178.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T05:31:07,093 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:07,093 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136., storeName=b07d9cd4230ca8093f260ec7e989e136/info, priority=13, startTime=1733463067055; duration=0sec
2024-12-06T05:31:07,093 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:31:07,093 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b07d9cd4230ca8093f260ec7e989e136:info
2024-12-06T05:31:07,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:07,957 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:08,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:08,958 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:31:09,074 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b07d9cd4230ca8093f260ec7e989e136 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T05:31:09,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/23006ea0a57e4d9aa1b8494095db3265 is 1080, key is row0221/info:/1733463067058/Put/seqid=0
2024-12-06T05:31:09,092 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741874_1050 (size=12523)
2024-12-06T05:31:09,092 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741874_1050 (size=12523)
2024-12-06T05:31:09,122 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=b07d9cd4230ca8093f260ec7e989e136, server=ec46afd49254,42957,1733463033291
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-12-06T05:31:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.3:58284 deadline: 1733463079121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=b07d9cd4230ca8093f260ec7e989e136, server=ec46afd49254,42957,1733463033291
2024-12-06T05:31:09,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136., hostname=ec46afd49254,42957,1733463033291, seqNum=131 , the old value is region=TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136., hostname=ec46afd49254,42957,1733463033291, seqNum=131, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=b07d9cd4230ca8093f260ec7e989e136, server=ec46afd49254,42957,1733463033291
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-06T05:31:09,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136., hostname=ec46afd49254,42957,1733463033291, seqNum=131 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=b07d9cd4230ca8093f260ec7e989e136, server=ec46afd49254,42957,1733463033291
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-06T05:31:09,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136., hostname=ec46afd49254,42957,1733463033291, seqNum=131 because the exception is null or not the one we care about
2024-12-06T05:31:09,493 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/23006ea0a57e4d9aa1b8494095db3265
2024-12-06T05:31:09,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/23006ea0a57e4d9aa1b8494095db3265 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/23006ea0a57e4d9aa1b8494095db3265
2024-12-06T05:31:09,505 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/23006ea0a57e4d9aa1b8494095db3265, entries=7, sequenceid=303, filesize=12.2 K
2024-12-06T05:31:09,506 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for b07d9cd4230ca8093f260ec7e989e136 in 432ms, sequenceid=303, compaction requested=true
2024-12-06T05:31:09,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:09,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b07d9cd4230ca8093f260ec7e989e136:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T05:31:09,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:31:09,506 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T05:31:09,507 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 195061 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T05:31:09,508 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1541): b07d9cd4230ca8093f260ec7e989e136/info is initiating minor compaction (all files)
2024-12-06T05:31:09,508 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b07d9cd4230ca8093f260ec7e989e136/info in TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.
2024-12-06T05:31:09,508 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/1db50090dbda45208968667793dfb5da, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/81c703cf89d547b6a6ceed3123401fe5, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/23006ea0a57e4d9aa1b8494095db3265] into tmpdir=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp, totalSize=190.5 K
2024-12-06T05:31:09,508 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1db50090dbda45208968667793dfb5da, keycount=146, bloomtype=ROW, size=159.7 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733463046604
2024-12-06T05:31:09,508 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81c703cf89d547b6a6ceed3123401fe5, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733463067031
2024-12-06T05:31:09,509 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23006ea0a57e4d9aa1b8494095db3265, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733463067058
2024-12-06T05:31:09,521 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b07d9cd4230ca8093f260ec7e989e136#info#compaction#89 average throughput is 56.78 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T05:31:09,522 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/bbfb2319af904221937758962e7c9a93 is 1080, key is row0062/info:/1733463046604/Put/seqid=0
2024-12-06T05:31:09,524 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741875_1051 (size=185211)
2024-12-06T05:31:09,524 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741875_1051 (size=185211)
2024-12-06T05:31:09,530 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/bbfb2319af904221937758962e7c9a93 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/bbfb2319af904221937758962e7c9a93
2024-12-06T05:31:09,536 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b07d9cd4230ca8093f260ec7e989e136/info of b07d9cd4230ca8093f260ec7e989e136 into bbfb2319af904221937758962e7c9a93(size=180.9 K), total size for store is 180.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T05:31:09,536 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:09,536 INFO  [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136., storeName=b07d9cd4230ca8093f260ec7e989e136/info, priority=13, startTime=1733463069506; duration=0sec
2024-12-06T05:31:09,536 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T05:31:09,536 DEBUG [RS:0;ec46afd49254:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b07d9cd4230ca8093f260ec7e989e136:info
2024-12-06T05:31:09,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:09,959 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:10,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:10,959 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:11,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:11,960 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:12,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:12,961 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:13,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:13,961 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:14,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:14,787 INFO  [master/ec46afd49254:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-06T05:31:14,787 INFO  [master/ec46afd49254:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-06T05:31:14,962 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:15,787 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:15,962 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:16,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:16,963 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:17,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:17,964 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:18,789 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:18,964 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:31:19,138 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b07d9cd4230ca8093f260ec7e989e136 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-06T05:31:19,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/5ceeffa5e7db4672879e8f3c1926b381 is 1080, key is row0228/info:/1733463069076/Put/seqid=0
2024-12-06T05:31:19,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741876_1052 (size=29807)
2024-12-06T05:31:19,159 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741876_1052 (size=29807)
2024-12-06T05:31:19,159 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/5ceeffa5e7db4672879e8f3c1926b381
2024-12-06T05:31:19,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/5ceeffa5e7db4672879e8f3c1926b381 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/5ceeffa5e7db4672879e8f3c1926b381
2024-12-06T05:31:19,168 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/5ceeffa5e7db4672879e8f3c1926b381, entries=23, sequenceid=330, filesize=29.1 K
2024-12-06T05:31:19,169 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=6.30 KB/6456 for b07d9cd4230ca8093f260ec7e989e136 in 31ms, sequenceid=330, compaction requested=false
2024-12-06T05:31:19,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:19,248 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20340
2024-12-06T05:31:19,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:19,965 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:20,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:20,966 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:21,151 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-12-06T05:31:21,151 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C42957%2C1733463033291.1733463081151
2024-12-06T05:31:21,167 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:21,167 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:21,167 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:21,168 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:21,168 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:21,168 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/WALs/ec46afd49254,42957,1733463033291/ec46afd49254%2C42957%2C1733463033291.1733463033909 with entries=315, filesize=309.33 KB; new WAL /user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/WALs/ec46afd49254,42957,1733463033291/ec46afd49254%2C42957%2C1733463033291.1733463081151
2024-12-06T05:31:21,169 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741833_1009 (size=316763)
2024-12-06T05:31:21,169 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741833_1009 (size=316763)
2024-12-06T05:31:21,178 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36837:36837),(127.0.0.1/127.0.0.1:35415:35415)]
2024-12-06T05:31:21,183 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b02b1ef6591d9f4c0384547bbc849d53:

2024-12-06T05:31:21,183 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB
2024-12-06T05:31:21,187 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/.tmp/info/e06fc687dcd043448caef47af53c20c5 is 193, key is TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136./info:regioninfo/1733463049649/Put/seqid=0
2024-12-06T05:31:21,201 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741878_1054 (size=6223)
2024-12-06T05:31:21,202 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741878_1054 (size=6223)
2024-12-06T05:31:21,202 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/.tmp/info/e06fc687dcd043448caef47af53c20c5
2024-12-06T05:31:21,208 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/.tmp/info/e06fc687dcd043448caef47af53c20c5 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/info/e06fc687dcd043448caef47af53c20c5
2024-12-06T05:31:21,214 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/info/e06fc687dcd043448caef47af53c20c5, entries=5, sequenceid=21, filesize=6.1 K
2024-12-06T05:31:21,215 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=21, compaction requested=false
2024-12-06T05:31:21,215 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-06T05:31:21,215 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing b07d9cd4230ca8093f260ec7e989e136 1/1 column families, dataSize=6.30 KB heapSize=7 KB
2024-12-06T05:31:21,220 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/a6048a68aba6485b8306d1af533c5b76 is 1080, key is row0251/info:/1733463079140/Put/seqid=0
2024-12-06T05:31:21,226 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741879_1055 (size=11436)
2024-12-06T05:31:21,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741879_1055 (size=11436)
2024-12-06T05:31:21,627 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.30 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/a6048a68aba6485b8306d1af533c5b76
2024-12-06T05:31:21,633 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/.tmp/info/a6048a68aba6485b8306d1af533c5b76 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/a6048a68aba6485b8306d1af533c5b76
2024-12-06T05:31:21,638 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/a6048a68aba6485b8306d1af533c5b76, entries=6, sequenceid=339, filesize=11.2 K
2024-12-06T05:31:21,639 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~6.30 KB/6456, heapSize ~6.98 KB/7152, currentSize=0 B/0 for b07d9cd4230ca8093f260ec7e989e136 in 424ms, sequenceid=339, compaction requested=true
2024-12-06T05:31:21,639 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b07d9cd4230ca8093f260ec7e989e136:

2024-12-06T05:31:21,639 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C42957%2C1733463033291.1733463081639
2024-12-06T05:31:21,645 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:21,645 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:21,645 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:21,645 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:21,645 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:21,645 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/WALs/ec46afd49254,42957,1733463033291/ec46afd49254%2C42957%2C1733463033291.1733463081151 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/WALs/ec46afd49254,42957,1733463033291/ec46afd49254%2C42957%2C1733463033291.1733463081639
2024-12-06T05:31:21,646 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35415:35415),(127.0.0.1/127.0.0.1:36837:36837)]
2024-12-06T05:31:21,646 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/WALs/ec46afd49254,42957,1733463033291/ec46afd49254%2C42957%2C1733463033291.1733463081151 is not closed yet, will try archiving it next time
2024-12-06T05:31:21,646 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741877_1053 (size=731)
2024-12-06T05:31:21,646 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741877_1053 (size=731)
2024-12-06T05:31:21,648 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/WALs/ec46afd49254,42957,1733463033291/ec46afd49254%2C42957%2C1733463033291.1733463033909 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/oldWALs/ec46afd49254%2C42957%2C1733463033291.1733463033909
2024-12-06T05:31:21,649 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-12-06T05:31:21,649 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/WALs/ec46afd49254,42957,1733463033291/ec46afd49254%2C42957%2C1733463033291.1733463081151 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/oldWALs/ec46afd49254%2C42957%2C1733463033291.1733463081151
2024-12-06T05:31:21,749 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-06T05:31:21,749 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T05:31:21,749 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:31:21,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:31:21,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:31:21,750 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-06T05:31:21,750 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-06T05:31:21,750 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=777264955, stopped=false
2024-12-06T05:31:21,750 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ec46afd49254,44883,1733463033114
2024-12-06T05:31:21,791 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:21,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T05:31:21,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T05:31:21,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:21,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:21,826 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T05:31:21,827 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T05:31:21,827 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:31:21,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:31:21,827 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ec46afd49254,42957,1733463033291' *****
2024-12-06T05:31:21,827 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-06T05:31:21,827 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:31:21,827 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-06T05:31:21,827 INFO  [RS:0;ec46afd49254:42957 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-06T05:31:21,827 INFO  [RS:0;ec46afd49254:42957 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-06T05:31:21,828 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-06T05:31:21,828 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer(3091): Received CLOSE for b02b1ef6591d9f4c0384547bbc849d53
2024-12-06T05:31:21,828 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer(3091): Received CLOSE for b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:31:21,828 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b02b1ef6591d9f4c0384547bbc849d53, disabling compactions & flushes
2024-12-06T05:31:21,828 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer(959): stopping server ec46afd49254,42957,1733463033291
2024-12-06T05:31:21,828 INFO  [RS:0;ec46afd49254:42957 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T05:31:21,828 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.
2024-12-06T05:31:21,828 INFO  [RS:0;ec46afd49254:42957 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ec46afd49254:42957.
2024-12-06T05:31:21,828 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.
2024-12-06T05:31:21,828 DEBUG [RS:0;ec46afd49254:42957 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:31:21,828 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53. after waiting 0 ms
2024-12-06T05:31:21,828 DEBUG [RS:0;ec46afd49254:42957 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:31:21,828 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.
2024-12-06T05:31:21,828 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-06T05:31:21,828 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-06T05:31:21,828 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-06T05:31:21,828 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-06T05:31:21,828 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-12-06T05:31:21,828 DEBUG [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer(1325): Online Regions={b02b1ef6591d9f4c0384547bbc849d53=TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53., 1588230740=hbase:meta,,1.1588230740, b07d9cd4230ca8093f260ec7e989e136=TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.}
2024-12-06T05:31:21,829 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:31:21,829 DEBUG [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, b02b1ef6591d9f4c0384547bbc849d53, b07d9cd4230ca8093f260ec7e989e136
2024-12-06T05:31:21,828 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b02b1ef6591d9f4c0384547bbc849d53/info/ede1a7be0f7648ed9e26ece8a508dcdc.5206502d6b4766aaa207f7e6fc8fc803->hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/ede1a7be0f7648ed9e26ece8a508dcdc-bottom] to archive
2024-12-06T05:31:21,829 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T05:31:21,829 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T05:31:21,829 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T05:31:21,829 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T05:31:21,829 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T05:31:21,829 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-06T05:31:21,831 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b02b1ef6591d9f4c0384547bbc849d53/info/ede1a7be0f7648ed9e26ece8a508dcdc.5206502d6b4766aaa207f7e6fc8fc803 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b02b1ef6591d9f4c0384547bbc849d53/info/ede1a7be0f7648ed9e26ece8a508dcdc.5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:31:21,831 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ec46afd49254:44883 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-06T05:31:21,831 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-12-06T05:31:21,836 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b02b1ef6591d9f4c0384547bbc849d53/recovered.edits/135.seqid, newMaxSeqId=135, maxSeqId=130
2024-12-06T05:31:21,836 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.
2024-12-06T05:31:21,836 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b02b1ef6591d9f4c0384547bbc849d53:
Waiting for close lock at 1733463081828Running coprocessor pre-close hooks at 1733463081828Disabling compacts and flushes for region at 1733463081828Disabling writes for close at 1733463081828Writing region close event to WAL at 1733463081832 (+4 ms)Running coprocessor post-close hooks at 1733463081836 (+4 ms)Closed at 1733463081836
2024-12-06T05:31:21,837 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1733463048719.b02b1ef6591d9f4c0384547bbc849d53.
2024-12-06T05:31:21,837 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b07d9cd4230ca8093f260ec7e989e136, disabling compactions & flushes
2024-12-06T05:31:21,837 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.
2024-12-06T05:31:21,837 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.
2024-12-06T05:31:21,837 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136. after waiting 0 ms
2024-12-06T05:31:21,837 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.
2024-12-06T05:31:21,837 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ede1a7be0f7648ed9e26ece8a508dcdc.5206502d6b4766aaa207f7e6fc8fc803->hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/5206502d6b4766aaa207f7e6fc8fc803/info/ede1a7be0f7648ed9e26ece8a508dcdc-top, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/TestLogRolling-testLogRolling=5206502d6b4766aaa207f7e6fc8fc803-31e93d720cb245e3b7274624b8aa3979, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ff82c6853ee647ec9e8edaf264164b25, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/TestLogRolling-testLogRolling=5206502d6b4766aaa207f7e6fc8fc803-c7fd43bdeb4c4c4a99e75c4f14fa56d3, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/d3e4d2aa9ea444afb7591242d7a4e19e, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/b8b752f870944dd994c604a8ce6d31d6, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/9a4b1754875f41c7a359c8ae94ba1845, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/51b242d83d014b8a9a00e86af0e23531, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/c262906748104c61a0cc6f715625b53d, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/faaded2782ef4a86a2dc27fe05868e3e, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/4236d541fa1e4a2ba5111a09e45d04eb, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/3ef6cc81732846c1991c6cba2cb6185f, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ff726c570bf84b2783edec2e8bb71708, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ff7de05de208462a84d20e4678bcb65b, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/38a7bfa312db49da8f1dc8da82976879, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/f4fbd942eca342409f75de7301fb2351, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/3a043bb4e8c84f108308e09e4ecb9778, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/1db50090dbda45208968667793dfb5da, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/47364b61a3f04c14ac719fa1f36411b6, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/81c703cf89d547b6a6ceed3123401fe5, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/23006ea0a57e4d9aa1b8494095db3265] to archive
2024-12-06T05:31:21,838 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-06T05:31:21,839 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ede1a7be0f7648ed9e26ece8a508dcdc.5206502d6b4766aaa207f7e6fc8fc803 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ede1a7be0f7648ed9e26ece8a508dcdc.5206502d6b4766aaa207f7e6fc8fc803
2024-12-06T05:31:21,840 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-12-06T05:31:21,840 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/TestLogRolling-testLogRolling=5206502d6b4766aaa207f7e6fc8fc803-31e93d720cb245e3b7274624b8aa3979 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/TestLogRolling-testLogRolling=5206502d6b4766aaa207f7e6fc8fc803-31e93d720cb245e3b7274624b8aa3979
2024-12-06T05:31:21,841 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T05:31:21,841 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T05:31:21,841 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733463081829Running coprocessor pre-close hooks at 1733463081829Disabling compacts and flushes for region at 1733463081829Disabling writes for close at 1733463081829Writing region close event to WAL at 1733463081831 (+2 ms)Running coprocessor post-close hooks at 1733463081841 (+10 ms)Closed at 1733463081841
2024-12-06T05:31:21,841 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-06T05:31:21,842 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ff82c6853ee647ec9e8edaf264164b25 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ff82c6853ee647ec9e8edaf264164b25
2024-12-06T05:31:21,842 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/TestLogRolling-testLogRolling=5206502d6b4766aaa207f7e6fc8fc803-c7fd43bdeb4c4c4a99e75c4f14fa56d3 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/TestLogRolling-testLogRolling=5206502d6b4766aaa207f7e6fc8fc803-c7fd43bdeb4c4c4a99e75c4f14fa56d3
2024-12-06T05:31:21,843 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/d3e4d2aa9ea444afb7591242d7a4e19e to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/d3e4d2aa9ea444afb7591242d7a4e19e
2024-12-06T05:31:21,844 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/b8b752f870944dd994c604a8ce6d31d6 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/b8b752f870944dd994c604a8ce6d31d6
2024-12-06T05:31:21,845 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/9a4b1754875f41c7a359c8ae94ba1845 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/9a4b1754875f41c7a359c8ae94ba1845
2024-12-06T05:31:21,846 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/51b242d83d014b8a9a00e86af0e23531 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/51b242d83d014b8a9a00e86af0e23531
2024-12-06T05:31:21,847 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/c262906748104c61a0cc6f715625b53d to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/c262906748104c61a0cc6f715625b53d
2024-12-06T05:31:21,848 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/faaded2782ef4a86a2dc27fe05868e3e to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/faaded2782ef4a86a2dc27fe05868e3e
2024-12-06T05:31:21,850 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/4236d541fa1e4a2ba5111a09e45d04eb to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/4236d541fa1e4a2ba5111a09e45d04eb
2024-12-06T05:31:21,851 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/3ef6cc81732846c1991c6cba2cb6185f to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/3ef6cc81732846c1991c6cba2cb6185f
2024-12-06T05:31:21,852 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ff726c570bf84b2783edec2e8bb71708 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ff726c570bf84b2783edec2e8bb71708
2024-12-06T05:31:21,853 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ff7de05de208462a84d20e4678bcb65b to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/ff7de05de208462a84d20e4678bcb65b
2024-12-06T05:31:21,854 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/38a7bfa312db49da8f1dc8da82976879 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/38a7bfa312db49da8f1dc8da82976879
2024-12-06T05:31:21,855 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/f4fbd942eca342409f75de7301fb2351 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/f4fbd942eca342409f75de7301fb2351
2024-12-06T05:31:21,857 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/3a043bb4e8c84f108308e09e4ecb9778 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/3a043bb4e8c84f108308e09e4ecb9778
2024-12-06T05:31:21,858 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/1db50090dbda45208968667793dfb5da to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/1db50090dbda45208968667793dfb5da
2024-12-06T05:31:21,859 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/47364b61a3f04c14ac719fa1f36411b6 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/47364b61a3f04c14ac719fa1f36411b6
2024-12-06T05:31:21,860 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/81c703cf89d547b6a6ceed3123401fe5 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/81c703cf89d547b6a6ceed3123401fe5
2024-12-06T05:31:21,861 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/23006ea0a57e4d9aa1b8494095db3265 to hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/archive/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/info/23006ea0a57e4d9aa1b8494095db3265
2024-12-06T05:31:21,861 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [ff82c6853ee647ec9e8edaf264164b25=42984, d3e4d2aa9ea444afb7591242d7a4e19e=12516, b8b752f870944dd994c604a8ce6d31d6=66870, 9a4b1754875f41c7a359c8ae94ba1845=21156, 51b242d83d014b8a9a00e86af0e23531=16828, c262906748104c61a0cc6f715625b53d=91843, faaded2782ef4a86a2dc27fe05868e3e=17906, 4236d541fa1e4a2ba5111a09e45d04eb=17906, 3ef6cc81732846c1991c6cba2cb6185f=113509, ff726c570bf84b2783edec2e8bb71708=13594, ff7de05de208462a84d20e4678bcb65b=17906, 38a7bfa312db49da8f1dc8da82976879=140766, f4fbd942eca342409f75de7301fb2351=19000, 3a043bb4e8c84f108308e09e4ecb9778=12517, 1db50090dbda45208968667793dfb5da=163525, 47364b61a3f04c14ac719fa1f36411b6=20092, 81c703cf89d547b6a6ceed3123401fe5=19013, 23006ea0a57e4d9aa1b8494095db3265=12523]
2024-12-06T05:31:21,862 INFO  [regionserver/ec46afd49254:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-06T05:31:21,862 INFO  [regionserver/ec46afd49254:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-06T05:31:21,865 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/data/default/TestLogRolling-testLogRolling/b07d9cd4230ca8093f260ec7e989e136/recovered.edits/342.seqid, newMaxSeqId=342, maxSeqId=130
2024-12-06T05:31:21,866 INFO  [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.
2024-12-06T05:31:21,866 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b07d9cd4230ca8093f260ec7e989e136:
Waiting for close lock at 1733463081837Running coprocessor pre-close hooks at 1733463081837Disabling compacts and flushes for region at 1733463081837Disabling writes for close at 1733463081837Writing region close event to WAL at 1733463081862 (+25 ms)Running coprocessor post-close hooks at 1733463081866 (+4 ms)Closed at 1733463081866
2024-12-06T05:31:21,866 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1733463048719.b07d9cd4230ca8093f260ec7e989e136.
2024-12-06T05:31:21,966 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:22,029 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer(976): stopping server ec46afd49254,42957,1733463033291; all regions closed.
2024-12-06T05:31:22,029 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:22,029 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:22,030 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:22,030 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:22,030 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:22,032 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741834_1010 (size=8107)
2024-12-06T05:31:22,032 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741834_1010 (size=8107)
2024-12-06T05:31:22,034 DEBUG [RS:0;ec46afd49254:42957 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/oldWALs
2024-12-06T05:31:22,035 INFO  [RS:0;ec46afd49254:42957 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ec46afd49254%2C42957%2C1733463033291.meta:.meta(num 1733463034233)
2024-12-06T05:31:22,035 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:22,035 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:22,035 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:22,035 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:22,035 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:22,037 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741880_1056 (size=780)
2024-12-06T05:31:22,037 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741880_1056 (size=780)
2024-12-06T05:31:22,039 DEBUG [RS:0;ec46afd49254:42957 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/oldWALs
2024-12-06T05:31:22,039 INFO  [RS:0;ec46afd49254:42957 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ec46afd49254%2C42957%2C1733463033291:(num 1733463081639)
2024-12-06T05:31:22,039 DEBUG [RS:0;ec46afd49254:42957 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:31:22,039 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T05:31:22,040 INFO  [RS:0;ec46afd49254:42957 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T05:31:22,040 INFO  [RS:0;ec46afd49254:42957 {}] hbase.ChoreService(370): Chore service for: regionserver/ec46afd49254:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-06T05:31:22,040 INFO  [RS:0;ec46afd49254:42957 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T05:31:22,040 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T05:31:22,040 INFO  [RS:0;ec46afd49254:42957 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42957
2024-12-06T05:31:22,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec46afd49254,42957,1733463033291
2024-12-06T05:31:22,058 INFO  [RS:0;ec46afd49254:42957 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T05:31:22,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T05:31:22,068 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec46afd49254,42957,1733463033291]
2024-12-06T05:31:22,079 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ec46afd49254,42957,1733463033291 already deleted, retry=false
2024-12-06T05:31:22,079 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ec46afd49254,42957,1733463033291 expired; onlineServers=0
2024-12-06T05:31:22,079 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ec46afd49254,44883,1733463033114' *****
2024-12-06T05:31:22,079 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-06T05:31:22,079 INFO  [M:0;ec46afd49254:44883 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T05:31:22,079 INFO  [M:0;ec46afd49254:44883 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T05:31:22,079 DEBUG [M:0;ec46afd49254:44883 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-06T05:31:22,079 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-06T05:31:22,079 DEBUG [M:0;ec46afd49254:44883 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-06T05:31:22,079 DEBUG [master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733463033616 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733463033616,5,FailOnTimeoutGroup]
2024-12-06T05:31:22,079 DEBUG [master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733463033616 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733463033616,5,FailOnTimeoutGroup]
2024-12-06T05:31:22,080 INFO  [M:0;ec46afd49254:44883 {}] hbase.ChoreService(370): Chore service for: master/ec46afd49254:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-06T05:31:22,080 INFO  [M:0;ec46afd49254:44883 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T05:31:22,080 DEBUG [M:0;ec46afd49254:44883 {}] master.HMaster(1795): Stopping service threads
2024-12-06T05:31:22,080 INFO  [M:0;ec46afd49254:44883 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-06T05:31:22,080 INFO  [M:0;ec46afd49254:44883 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T05:31:22,080 INFO  [M:0;ec46afd49254:44883 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-06T05:31:22,080 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-06T05:31:22,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-06T05:31:22,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:22,089 DEBUG [M:0;ec46afd49254:44883 {}] zookeeper.ZKUtil(347): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-06T05:31:22,089 WARN  [M:0;ec46afd49254:44883 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-06T05:31:22,090 INFO  [M:0;ec46afd49254:44883 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/.lastflushedseqids
2024-12-06T05:31:22,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741881_1057 (size=228)
2024-12-06T05:31:22,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741881_1057 (size=228)
2024-12-06T05:31:22,114 INFO  [M:0;ec46afd49254:44883 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-06T05:31:22,114 INFO  [M:0;ec46afd49254:44883 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-06T05:31:22,114 DEBUG [M:0;ec46afd49254:44883 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T05:31:22,114 INFO  [M:0;ec46afd49254:44883 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:31:22,114 DEBUG [M:0;ec46afd49254:44883 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:31:22,114 DEBUG [M:0;ec46afd49254:44883 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T05:31:22,114 DEBUG [M:0;ec46afd49254:44883 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:31:22,115 INFO  [M:0;ec46afd49254:44883 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.43 KB heapSize=63.38 KB
2024-12-06T05:31:22,131 DEBUG [M:0;ec46afd49254:44883 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/58df4b9ff1f64328be4f535bc874b5ab is 82, key is hbase:meta,,1/info:regioninfo/1733463034265/Put/seqid=0
2024-12-06T05:31:22,136 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741882_1058 (size=5672)
2024-12-06T05:31:22,136 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741882_1058 (size=5672)
2024-12-06T05:31:22,137 INFO  [M:0;ec46afd49254:44883 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/58df4b9ff1f64328be4f535bc874b5ab
2024-12-06T05:31:22,157 DEBUG [M:0;ec46afd49254:44883 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dd576a1c05e6418bb6171a8691db4155 is 750, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733463034802/Put/seqid=0
2024-12-06T05:31:22,169 INFO  [RS:0;ec46afd49254:42957 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T05:31:22,169 INFO  [RS:0;ec46afd49254:42957 {}] regionserver.HRegionServer(1031): Exiting; stopping=ec46afd49254,42957,1733463033291; zookeeper connection closed.
2024-12-06T05:31:22,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:31:22,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101a91ec0b30001, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:31:22,169 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@35bb88d2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@35bb88d2
2024-12-06T05:31:22,169 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-06T05:31:22,170 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741883_1059 (size=7090)
2024-12-06T05:31:22,170 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741883_1059 (size=7090)
2024-12-06T05:31:22,171 INFO  [M:0;ec46afd49254:44883 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.83 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dd576a1c05e6418bb6171a8691db4155
2024-12-06T05:31:22,175 INFO  [M:0;ec46afd49254:44883 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dd576a1c05e6418bb6171a8691db4155
2024-12-06T05:31:22,195 DEBUG [M:0;ec46afd49254:44883 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a7abc2f2233e45918ec38077138e3c30 is 69, key is ec46afd49254,42957,1733463033291/rs:state/1733463033750/Put/seqid=0
2024-12-06T05:31:22,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741884_1060 (size=5156)
2024-12-06T05:31:22,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741884_1060 (size=5156)
2024-12-06T05:31:22,200 INFO  [M:0;ec46afd49254:44883 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a7abc2f2233e45918ec38077138e3c30
2024-12-06T05:31:22,219 DEBUG [M:0;ec46afd49254:44883 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5c99f3dd7a9b44a0b3e8b6ed6c1a60f4 is 52, key is load_balancer_on/state:d/1733463034425/Put/seqid=0
2024-12-06T05:31:22,224 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741885_1061 (size=5056)
2024-12-06T05:31:22,224 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741885_1061 (size=5056)
2024-12-06T05:31:22,225 INFO  [M:0;ec46afd49254:44883 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5c99f3dd7a9b44a0b3e8b6ed6c1a60f4
2024-12-06T05:31:22,231 DEBUG [M:0;ec46afd49254:44883 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/58df4b9ff1f64328be4f535bc874b5ab as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/58df4b9ff1f64328be4f535bc874b5ab
2024-12-06T05:31:22,236 INFO  [M:0;ec46afd49254:44883 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/58df4b9ff1f64328be4f535bc874b5ab, entries=8, sequenceid=125, filesize=5.5 K
2024-12-06T05:31:22,237 DEBUG [M:0;ec46afd49254:44883 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dd576a1c05e6418bb6171a8691db4155 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dd576a1c05e6418bb6171a8691db4155
2024-12-06T05:31:22,242 INFO  [M:0;ec46afd49254:44883 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dd576a1c05e6418bb6171a8691db4155
2024-12-06T05:31:22,242 INFO  [M:0;ec46afd49254:44883 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dd576a1c05e6418bb6171a8691db4155, entries=13, sequenceid=125, filesize=6.9 K
2024-12-06T05:31:22,243 DEBUG [M:0;ec46afd49254:44883 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a7abc2f2233e45918ec38077138e3c30 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a7abc2f2233e45918ec38077138e3c30
2024-12-06T05:31:22,248 INFO  [M:0;ec46afd49254:44883 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a7abc2f2233e45918ec38077138e3c30, entries=1, sequenceid=125, filesize=5.0 K
2024-12-06T05:31:22,249 DEBUG [M:0;ec46afd49254:44883 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5c99f3dd7a9b44a0b3e8b6ed6c1a60f4 as hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5c99f3dd7a9b44a0b3e8b6ed6c1a60f4
2024-12-06T05:31:22,255 INFO  [M:0;ec46afd49254:44883 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33745/user/jenkins/test-data/91221af0-a8dd-4288-8e2b-f2049a86a1da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5c99f3dd7a9b44a0b3e8b6ed6c1a60f4, entries=1, sequenceid=125, filesize=4.9 K
2024-12-06T05:31:22,256 INFO  [M:0;ec46afd49254:44883 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.32 KB/64840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 141ms, sequenceid=125, compaction requested=false
2024-12-06T05:31:22,258 INFO  [M:0;ec46afd49254:44883 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:31:22,258 DEBUG [M:0;ec46afd49254:44883 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733463082114Disabling compacts and flushes for region at 1733463082114Disabling writes for close at 1733463082114Obtaining lock to block concurrent updates at 1733463082115 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733463082115Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52663, getHeapSize=64840, getOffHeapSize=0, getCellsCount=148 at 1733463082115Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733463082116 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733463082116Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733463082131 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733463082131Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733463082141 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733463082157 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733463082157Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733463082175 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733463082194 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733463082194Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733463082204 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733463082218 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733463082218Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13ecc15c: reopening flushed file at 1733463082230 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51ae5eb0: reopening flushed file at 1733463082237 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c1c8f7c: reopening flushed file at 1733463082242 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7939281: reopening flushed file at 1733463082248 (+6 ms)Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.32 KB/64840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 141ms, sequenceid=125, compaction requested=false at 1733463082256 (+8 ms)Writing region close event to WAL at 1733463082258 (+2 ms)Closed at 1733463082258
2024-12-06T05:31:22,258 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:22,258 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:22,258 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:22,259 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:22,259 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:22,260 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40111 is added to blk_1073741830_1006 (size=61332)
2024-12-06T05:31:22,260 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741830_1006 (size=61332)
2024-12-06T05:31:22,266 INFO  [M:0;ec46afd49254:44883 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-06T05:31:22,266 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T05:31:22,266 INFO  [M:0;ec46afd49254:44883 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44883
2024-12-06T05:31:22,267 INFO  [M:0;ec46afd49254:44883 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T05:31:22,379 INFO  [M:0;ec46afd49254:44883 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T05:31:22,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:31:22,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44883-0x101a91ec0b30000, quorum=127.0.0.1:57049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:31:22,382 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@42c5c09{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:31:22,382 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5ce0a24{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:31:22,382 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:31:22,382 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f61588{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:31:22,382 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13ef5561{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/hadoop.log.dir/,STOPPED}
2024-12-06T05:31:22,384 WARN  [BP-652329374-172.17.0.3-1733463030916 heartbeating to localhost/127.0.0.1:33745 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:31:22,384 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:31:22,384 WARN  [BP-652329374-172.17.0.3-1733463030916 heartbeating to localhost/127.0.0.1:33745 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-652329374-172.17.0.3-1733463030916 (Datanode Uuid 83a570a5-e71b-4269-9188-b090b003c8e9) service to localhost/127.0.0.1:33745
2024-12-06T05:31:22,384 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:31:22,385 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/cluster_aa9f0358-d841-213d-3fe9-e6bb995c275f/data/data3/current/BP-652329374-172.17.0.3-1733463030916 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:31:22,385 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/cluster_aa9f0358-d841-213d-3fe9-e6bb995c275f/data/data4/current/BP-652329374-172.17.0.3-1733463030916 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:31:22,385 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:31:22,387 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4a6e8e46{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:31:22,387 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1204fb24{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:31:22,388 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:31:22,388 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f424370{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:31:22,388 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2eb912ab{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/hadoop.log.dir/,STOPPED}
2024-12-06T05:31:22,389 WARN  [BP-652329374-172.17.0.3-1733463030916 heartbeating to localhost/127.0.0.1:33745 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:31:22,389 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:31:22,389 WARN  [BP-652329374-172.17.0.3-1733463030916 heartbeating to localhost/127.0.0.1:33745 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-652329374-172.17.0.3-1733463030916 (Datanode Uuid d7e846d0-9983-4fd8-93fc-fc883961243f) service to localhost/127.0.0.1:33745
2024-12-06T05:31:22,389 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:31:22,390 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/cluster_aa9f0358-d841-213d-3fe9-e6bb995c275f/data/data1/current/BP-652329374-172.17.0.3-1733463030916 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:31:22,390 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/cluster_aa9f0358-d841-213d-3fe9-e6bb995c275f/data/data2/current/BP-652329374-172.17.0.3-1733463030916 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:31:22,390 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:31:22,395 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5ff7780b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T05:31:22,396 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@20d96a0d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:31:22,396 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:31:22,396 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@425d5d71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:31:22,396 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@314e7370{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/hadoop.log.dir/,STOPPED}
2024-12-06T05:31:22,403 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-06T05:31:22,435 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-06T05:31:22,442 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=232 (was 205)
Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33745 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33745
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33745 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33745
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33745
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:33745
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33745 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33745
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:33745
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=503 (was 485) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=247 (was 254), ProcessCount=11 (was 11), AvailableMemoryMB=6311 (was 6454)
2024-12-06T05:31:22,451 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=232, OpenFileDescriptor=503, MaxFileDescriptor=1048576, SystemLoadAverage=259, ProcessCount=11, AvailableMemoryMB=6311
2024-12-06T05:31:22,451 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-06T05:31:22,451 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/hadoop.log.dir so I do NOT create it in target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb
2024-12-06T05:31:22,451 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fac95772-08a3-970e-b0dd-ddad63118ffe/hadoop.tmp.dir so I do NOT create it in target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb
2024-12-06T05:31:22,451 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/cluster_1f6d5f9e-3a16-da76-5706-ef7c8d453a05, deleteOnExit=true
2024-12-06T05:31:22,451 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-06T05:31:22,452 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/test.cache.data in system properties and HBase conf
2024-12-06T05:31:22,452 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/hadoop.tmp.dir in system properties and HBase conf
2024-12-06T05:31:22,452 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/hadoop.log.dir in system properties and HBase conf
2024-12-06T05:31:22,452 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-06T05:31:22,452 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-06T05:31:22,452 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-06T05:31:22,452 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-06T05:31:22,452 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-06T05:31:22,452 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-06T05:31:22,453 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-06T05:31:22,453 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T05:31:22,453 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-06T05:31:22,453 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-06T05:31:22,453 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T05:31:22,453 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T05:31:22,453 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-06T05:31:22,453 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/nfs.dump.dir in system properties and HBase conf
2024-12-06T05:31:22,453 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/java.io.tmpdir in system properties and HBase conf
2024-12-06T05:31:22,453 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T05:31:22,453 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-06T05:31:22,453 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-06T05:31:22,467 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T05:31:22,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:22,849 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:31:22,852 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:31:22,853 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:31:22,853 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:31:22,853 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T05:31:22,854 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:31:22,854 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b72d363{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:31:22,854 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4c66b7d1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:31:22,945 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@61edd007{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/java.io.tmpdir/jetty-localhost-36825-hadoop-hdfs-3_4_1-tests_jar-_-any-7003962963443148121/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T05:31:22,945 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7d664f93{HTTP/1.1, (http/1.1)}{localhost:36825}
2024-12-06T05:31:22,945 INFO  [Time-limited test {}] server.Server(415): Started @302557ms
2024-12-06T05:31:22,956 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T05:31:22,967 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:23,177 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:31:23,179 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:31:23,180 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:31:23,180 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:31:23,180 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T05:31:23,180 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fcd61c6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:31:23,181 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1040cecb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:31:23,283 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@9fc2daa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/java.io.tmpdir/jetty-localhost-40705-hadoop-hdfs-3_4_1-tests_jar-_-any-1839911395507698521/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:31:23,283 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2a1af98b{HTTP/1.1, (http/1.1)}{localhost:40705}
2024-12-06T05:31:23,283 INFO  [Time-limited test {}] server.Server(415): Started @302894ms
2024-12-06T05:31:23,284 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:31:23,310 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T05:31:23,313 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T05:31:23,316 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T05:31:23,316 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T05:31:23,316 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T05:31:23,317 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@49e6dd92{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/hadoop.log.dir/,AVAILABLE}
2024-12-06T05:31:23,317 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6bfe0bbd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T05:31:23,420 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@627a202d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/java.io.tmpdir/jetty-localhost-41527-hadoop-hdfs-3_4_1-tests_jar-_-any-2461149919767749429/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:31:23,421 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22ed154c{HTTP/1.1, (http/1.1)}{localhost:41527}
2024-12-06T05:31:23,421 INFO  [Time-limited test {}] server.Server(415): Started @303032ms
2024-12-06T05:31:23,421 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T05:31:23,776 INFO  [regionserver/ec46afd49254:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T05:31:23,793 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:23,967 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:24,423 WARN  [Thread-2498 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/cluster_1f6d5f9e-3a16-da76-5706-ef7c8d453a05/data/data2/current/BP-798507793-172.17.0.3-1733463082470/current, will proceed with Du for space computation calculation, 
2024-12-06T05:31:24,423 WARN  [Thread-2497 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/cluster_1f6d5f9e-3a16-da76-5706-ef7c8d453a05/data/data1/current/BP-798507793-172.17.0.3-1733463082470/current, will proceed with Du for space computation calculation, 
2024-12-06T05:31:24,453 WARN  [Thread-2461 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:31:24,458 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe306836b1f2cd4b3 with lease ID 0x396a6ece89af88a0: Processing first storage report for DS-da3eca7d-d751-44cc-900d-ec7a8d9a9631 from datanode DatanodeRegistration(127.0.0.1:46395, datanodeUuid=a5b3a820-c3bb-498c-a6ae-4a86bd48eae9, infoPort=44163, infoSecurePort=0, ipcPort=38007, storageInfo=lv=-57;cid=testClusterID;nsid=518096374;c=1733463082470)
2024-12-06T05:31:24,458 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe306836b1f2cd4b3 with lease ID 0x396a6ece89af88a0: from storage DS-da3eca7d-d751-44cc-900d-ec7a8d9a9631 node DatanodeRegistration(127.0.0.1:46395, datanodeUuid=a5b3a820-c3bb-498c-a6ae-4a86bd48eae9, infoPort=44163, infoSecurePort=0, ipcPort=38007, storageInfo=lv=-57;cid=testClusterID;nsid=518096374;c=1733463082470), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-06T05:31:24,458 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe306836b1f2cd4b3 with lease ID 0x396a6ece89af88a0: Processing first storage report for DS-42db98b4-87c4-43ef-bfa3-f2a9a1289630 from datanode DatanodeRegistration(127.0.0.1:46395, datanodeUuid=a5b3a820-c3bb-498c-a6ae-4a86bd48eae9, infoPort=44163, infoSecurePort=0, ipcPort=38007, storageInfo=lv=-57;cid=testClusterID;nsid=518096374;c=1733463082470)
2024-12-06T05:31:24,458 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe306836b1f2cd4b3 with lease ID 0x396a6ece89af88a0: from storage DS-42db98b4-87c4-43ef-bfa3-f2a9a1289630 node DatanodeRegistration(127.0.0.1:46395, datanodeUuid=a5b3a820-c3bb-498c-a6ae-4a86bd48eae9, infoPort=44163, infoSecurePort=0, ipcPort=38007, storageInfo=lv=-57;cid=testClusterID;nsid=518096374;c=1733463082470), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:31:24,550 WARN  [Thread-2509 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/cluster_1f6d5f9e-3a16-da76-5706-ef7c8d453a05/data/data3/current/BP-798507793-172.17.0.3-1733463082470/current, will proceed with Du for space computation calculation, 
2024-12-06T05:31:24,550 WARN  [Thread-2510 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/cluster_1f6d5f9e-3a16-da76-5706-ef7c8d453a05/data/data4/current/BP-798507793-172.17.0.3-1733463082470/current, will proceed with Du for space computation calculation, 
2024-12-06T05:31:24,565 WARN  [Thread-2484 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T05:31:24,567 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf7defba8f367635a with lease ID 0x396a6ece89af88a1: Processing first storage report for DS-f22caff2-b0af-4532-990d-df9b8121ed4f from datanode DatanodeRegistration(127.0.0.1:37533, datanodeUuid=35e86b08-28a6-4ae7-8b5f-4cf23ca5da70, infoPort=41105, infoSecurePort=0, ipcPort=36161, storageInfo=lv=-57;cid=testClusterID;nsid=518096374;c=1733463082470)
2024-12-06T05:31:24,567 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf7defba8f367635a with lease ID 0x396a6ece89af88a1: from storage DS-f22caff2-b0af-4532-990d-df9b8121ed4f node DatanodeRegistration(127.0.0.1:37533, datanodeUuid=35e86b08-28a6-4ae7-8b5f-4cf23ca5da70, infoPort=41105, infoSecurePort=0, ipcPort=36161, storageInfo=lv=-57;cid=testClusterID;nsid=518096374;c=1733463082470), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:31:24,567 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf7defba8f367635a with lease ID 0x396a6ece89af88a1: Processing first storage report for DS-7b97672c-21f9-46b8-8304-7940d4f33711 from datanode DatanodeRegistration(127.0.0.1:37533, datanodeUuid=35e86b08-28a6-4ae7-8b5f-4cf23ca5da70, infoPort=41105, infoSecurePort=0, ipcPort=36161, storageInfo=lv=-57;cid=testClusterID;nsid=518096374;c=1733463082470)
2024-12-06T05:31:24,567 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf7defba8f367635a with lease ID 0x396a6ece89af88a1: from storage DS-7b97672c-21f9-46b8-8304-7940d4f33711 node DatanodeRegistration(127.0.0.1:37533, datanodeUuid=35e86b08-28a6-4ae7-8b5f-4cf23ca5da70, infoPort=41105, infoSecurePort=0, ipcPort=36161, storageInfo=lv=-57;cid=testClusterID;nsid=518096374;c=1733463082470), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T05:31:24,659 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb
2024-12-06T05:31:24,661 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/cluster_1f6d5f9e-3a16-da76-5706-ef7c8d453a05/zookeeper_0, clientPort=56790, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/cluster_1f6d5f9e-3a16-da76-5706-ef7c8d453a05/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/cluster_1f6d5f9e-3a16-da76-5706-ef7c8d453a05/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-06T05:31:24,662 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56790
2024-12-06T05:31:24,663 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:31:24,664 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:31:24,671 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741825_1001 (size=7)
2024-12-06T05:31:24,672 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741825_1001 (size=7)
2024-12-06T05:31:24,673 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33 with version=8
2024-12-06T05:31:24,673 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:40559/user/jenkins/test-data/15360ca2-c6be-565f-a82e-94edf54b2ac1/hbase-staging
2024-12-06T05:31:24,674 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/ec46afd49254:0 server-side Connection retries=45
2024-12-06T05:31:24,675 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:31:24,675 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T05:31:24,675 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T05:31:24,675 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:31:24,675 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T05:31:24,675 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-06T05:31:24,675 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T05:31:24,675 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37087
2024-12-06T05:31:24,676 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37087 connecting to ZooKeeper ensemble=127.0.0.1:56790
2024-12-06T05:31:24,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:370870x0, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T05:31:24,741 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37087-0x101a91f8a200000 connected
2024-12-06T05:31:24,793 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:24,827 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:31:24,828 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:31:24,830 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:31:24,830 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33, hbase.cluster.distributed=false
2024-12-06T05:31:24,832 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T05:31:24,833 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37087
2024-12-06T05:31:24,833 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37087
2024-12-06T05:31:24,834 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37087
2024-12-06T05:31:24,834 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37087
2024-12-06T05:31:24,834 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37087
2024-12-06T05:31:24,850 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/ec46afd49254:0 server-side Connection retries=45
2024-12-06T05:31:24,850 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:31:24,850 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T05:31:24,850 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T05:31:24,850 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T05:31:24,850 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T05:31:24,850 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-06T05:31:24,850 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T05:31:24,851 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43181
2024-12-06T05:31:24,852 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43181 connecting to ZooKeeper ensemble=127.0.0.1:56790
2024-12-06T05:31:24,853 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:31:24,854 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:31:24,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:431810x0, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T05:31:24,869 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:431810x0, quorum=127.0.0.1:56790, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:31:24,869 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43181-0x101a91f8a200001 connected
2024-12-06T05:31:24,869 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-06T05:31:24,870 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-06T05:31:24,870 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-06T05:31:24,871 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T05:31:24,871 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43181
2024-12-06T05:31:24,872 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43181
2024-12-06T05:31:24,872 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43181
2024-12-06T05:31:24,876 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43181
2024-12-06T05:31:24,876 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43181
2024-12-06T05:31:24,887 DEBUG [M:0;ec46afd49254:37087 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ec46afd49254:37087
2024-12-06T05:31:24,888 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ec46afd49254,37087,1733463084674
2024-12-06T05:31:24,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:31:24,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:31:24,900 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ec46afd49254,37087,1733463084674
2024-12-06T05:31:24,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:24,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-06T05:31:24,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:24,911 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-06T05:31:24,911 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ec46afd49254,37087,1733463084674 from backup master directory
2024-12-06T05:31:24,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ec46afd49254,37087,1733463084674
2024-12-06T05:31:24,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:31:24,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T05:31:24,921 WARN  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T05:31:24,921 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ec46afd49254,37087,1733463084674
2024-12-06T05:31:24,925 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/hbase.id] with ID: 20172045-abb2-4205-82b9-e5db37da1c00
2024-12-06T05:31:24,925 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/.tmp/hbase.id
2024-12-06T05:31:24,930 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741826_1002 (size=42)
2024-12-06T05:31:24,930 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741826_1002 (size=42)
2024-12-06T05:31:24,931 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/.tmp/hbase.id]:[hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/hbase.id]
2024-12-06T05:31:24,941 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:31:24,941 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-06T05:31:24,942 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-06T05:31:24,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:24,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:24,958 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741827_1003 (size=196)
2024-12-06T05:31:24,958 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741827_1003 (size=196)
2024-12-06T05:31:24,959 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T05:31:24,960 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-06T05:31:24,960 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:31:24,968 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741828_1004 (size=1189)
2024-12-06T05:31:24,968 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741828_1004 (size=1189)
2024-12-06T05:31:24,968 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:24,969 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store
2024-12-06T05:31:24,975 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741829_1005 (size=34)
2024-12-06T05:31:24,975 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741829_1005 (size=34)
2024-12-06T05:31:24,976 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:31:24,976 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T05:31:24,976 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:31:24,976 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:31:24,976 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T05:31:24,976 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:31:24,976 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:31:24,976 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733463084976Disabling compacts and flushes for region at 1733463084976Disabling writes for close at 1733463084976Writing region close event to WAL at 1733463084976Closed at 1733463084976
2024-12-06T05:31:24,977 WARN  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/.initializing
2024-12-06T05:31:24,977 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/WALs/ec46afd49254,37087,1733463084674
2024-12-06T05:31:24,980 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C37087%2C1733463084674, suffix=, logDir=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/WALs/ec46afd49254,37087,1733463084674, archiveDir=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/oldWALs, maxLogs=10
2024-12-06T05:31:24,980 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C37087%2C1733463084674.1733463084980
2024-12-06T05:31:24,984 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/WALs/ec46afd49254,37087,1733463084674/ec46afd49254%2C37087%2C1733463084674.1733463084980
2024-12-06T05:31:24,986 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41105:41105),(127.0.0.1/127.0.0.1:44163:44163)]
2024-12-06T05:31:24,986 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:31:24,986 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:31:24,987 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:31:24,987 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:31:24,988 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:31:24,989 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-06T05:31:24,989 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:31:24,990 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:31:24,990 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:31:24,991 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-06T05:31:24,991 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:31:24,992 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:31:24,992 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:31:24,993 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-06T05:31:24,993 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:31:24,994 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:31:24,994 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T05:31:24,995 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-06T05:31:24,995 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:31:24,996 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T05:31:24,996 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:31:24,997 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:31:24,997 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:31:24,998 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:31:24,998 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:31:24,999 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-06T05:31:25,000 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T05:31:25,002 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:31:25,003 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=883811, jitterRate=0.12382496893405914}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-06T05:31:25,003 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733463084987Initializing all the Stores at 1733463084988 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463084988Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463084988Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463084988Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463084988Cleaning up temporary data from old regions at 1733463084998 (+10 ms)Region opened successfully at 1733463085003 (+5 ms)
2024-12-06T05:31:25,004 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-06T05:31:25,007 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8ff890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0
2024-12-06T05:31:25,008 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-06T05:31:25,009 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-06T05:31:25,009 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-06T05:31:25,009 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-06T05:31:25,009 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-06T05:31:25,010 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-06T05:31:25,010 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-06T05:31:25,012 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-06T05:31:25,013 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-06T05:31:25,026 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-06T05:31:25,026 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-06T05:31:25,027 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-06T05:31:25,037 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-06T05:31:25,037 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-06T05:31:25,038 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-06T05:31:25,047 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-06T05:31:25,048 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-06T05:31:25,058 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-06T05:31:25,061 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-06T05:31:25,068 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-06T05:31:25,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T05:31:25,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T05:31:25,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:25,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:25,079 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ec46afd49254,37087,1733463084674, sessionid=0x101a91f8a200000, setting cluster-up flag (Was=false)
2024-12-06T05:31:25,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:25,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:25,131 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-06T05:31:25,132 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec46afd49254,37087,1733463084674
2024-12-06T05:31:25,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:25,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:25,184 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-06T05:31:25,185 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec46afd49254,37087,1733463084674
2024-12-06T05:31:25,187 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-06T05:31:25,189 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-06T05:31:25,189 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-06T05:31:25,189 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-06T05:31:25,189 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ec46afd49254,37087,1733463084674
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-06T05:31:25,191 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:31:25,191 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:31:25,191 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:31:25,191 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5
2024-12-06T05:31:25,191 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ec46afd49254:0, corePoolSize=10, maxPoolSize=10
2024-12-06T05:31:25,191 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:31:25,192 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ec46afd49254:0, corePoolSize=2, maxPoolSize=2
2024-12-06T05:31:25,192 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:31:25,193 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733463115192
2024-12-06T05:31:25,193 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-06T05:31:25,193 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-06T05:31:25,193 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-06T05:31:25,193 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-06T05:31:25,193 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-06T05:31:25,193 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-06T05:31:25,193 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:25,193 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:31:25,194 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-06T05:31:25,194 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-06T05:31:25,194 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-06T05:31:25,194 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-06T05:31:25,194 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-06T05:31:25,194 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-06T05:31:25,194 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733463085194,5,FailOnTimeoutGroup]
2024-12-06T05:31:25,195 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733463085194,5,FailOnTimeoutGroup]
2024-12-06T05:31:25,195 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:31:25,195 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:25,195 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-06T05:31:25,195 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:25,195 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:25,195 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-06T05:31:25,216 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741831_1007 (size=1321)
2024-12-06T05:31:25,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741831_1007 (size=1321)
2024-12-06T05:31:25,217 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-06T05:31:25,217 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33
2024-12-06T05:31:25,223 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741832_1008 (size=32)
2024-12-06T05:31:25,223 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741832_1008 (size=32)
2024-12-06T05:31:25,223 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:31:25,224 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T05:31:25,225 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T05:31:25,225 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:31:25,226 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:31:25,226 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T05:31:25,227 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T05:31:25,227 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:31:25,227 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:31:25,227 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T05:31:25,228 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T05:31:25,228 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:31:25,228 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:31:25,228 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T05:31:25,229 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T05:31:25,229 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:31:25,230 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:31:25,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T05:31:25,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/data/hbase/meta/1588230740
2024-12-06T05:31:25,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/data/hbase/meta/1588230740
2024-12-06T05:31:25,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T05:31:25,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T05:31:25,232 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T05:31:25,233 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T05:31:25,235 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T05:31:25,235 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=881433, jitterRate=0.12080098688602448}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T05:31:25,236 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733463085224Initializing all the Stores at 1733463085224Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463085224Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463085224Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463085224Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463085224Cleaning up temporary data from old regions at 1733463085232 (+8 ms)Region opened successfully at 1733463085236 (+4 ms)
2024-12-06T05:31:25,236 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T05:31:25,236 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T05:31:25,236 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T05:31:25,236 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T05:31:25,236 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T05:31:25,236 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T05:31:25,236 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733463085236Disabling compacts and flushes for region at 1733463085236Disabling writes for close at 1733463085236Writing region close event to WAL at 1733463085236Closed at 1733463085236
2024-12-06T05:31:25,237 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:31:25,237 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-06T05:31:25,238 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-06T05:31:25,238 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T05:31:25,239 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-06T05:31:25,278 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.HRegionServer(746): ClusterId : 20172045-abb2-4205-82b9-e5db37da1c00
2024-12-06T05:31:25,278 DEBUG [RS:0;ec46afd49254:43181 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-06T05:31:25,290 DEBUG [RS:0;ec46afd49254:43181 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-06T05:31:25,290 DEBUG [RS:0;ec46afd49254:43181 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-06T05:31:25,301 DEBUG [RS:0;ec46afd49254:43181 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-06T05:31:25,301 DEBUG [RS:0;ec46afd49254:43181 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ca01280, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0
2024-12-06T05:31:25,314 DEBUG [RS:0;ec46afd49254:43181 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ec46afd49254:43181
2024-12-06T05:31:25,314 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-06T05:31:25,314 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-06T05:31:25,314 DEBUG [RS:0;ec46afd49254:43181 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-06T05:31:25,314 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec46afd49254,37087,1733463084674 with port=43181, startcode=1733463084850
2024-12-06T05:31:25,315 DEBUG [RS:0;ec46afd49254:43181 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-06T05:31:25,316 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50547, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-12-06T05:31:25,317 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37087 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ec46afd49254,43181,1733463084850
2024-12-06T05:31:25,317 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37087 {}] master.ServerManager(517): Registering regionserver=ec46afd49254,43181,1733463084850
2024-12-06T05:31:25,318 DEBUG [RS:0;ec46afd49254:43181 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33
2024-12-06T05:31:25,318 DEBUG [RS:0;ec46afd49254:43181 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40171
2024-12-06T05:31:25,318 DEBUG [RS:0;ec46afd49254:43181 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-06T05:31:25,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T05:31:25,330 DEBUG [RS:0;ec46afd49254:43181 {}] zookeeper.ZKUtil(111): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec46afd49254,43181,1733463084850
2024-12-06T05:31:25,330 WARN  [RS:0;ec46afd49254:43181 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T05:31:25,330 INFO  [RS:0;ec46afd49254:43181 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:31:25,330 DEBUG [RS:0;ec46afd49254:43181 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/WALs/ec46afd49254,43181,1733463084850
2024-12-06T05:31:25,330 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec46afd49254,43181,1733463084850]
2024-12-06T05:31:25,333 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-06T05:31:25,334 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-06T05:31:25,335 INFO  [RS:0;ec46afd49254:43181 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T05:31:25,335 INFO  [RS:0;ec46afd49254:43181 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:25,335 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-06T05:31:25,335 INFO  [RS:0;ec46afd49254:43181 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-06T05:31:25,336 INFO  [RS:0;ec46afd49254:43181 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:25,336 DEBUG [RS:0;ec46afd49254:43181 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:31:25,336 DEBUG [RS:0;ec46afd49254:43181 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:31:25,336 DEBUG [RS:0;ec46afd49254:43181 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:31:25,336 DEBUG [RS:0;ec46afd49254:43181 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:31:25,336 DEBUG [RS:0;ec46afd49254:43181 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:31:25,336 DEBUG [RS:0;ec46afd49254:43181 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0, corePoolSize=2, maxPoolSize=2
2024-12-06T05:31:25,336 DEBUG [RS:0;ec46afd49254:43181 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:31:25,336 DEBUG [RS:0;ec46afd49254:43181 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:31:25,336 DEBUG [RS:0;ec46afd49254:43181 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:31:25,336 DEBUG [RS:0;ec46afd49254:43181 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:31:25,336 DEBUG [RS:0;ec46afd49254:43181 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:31:25,336 DEBUG [RS:0;ec46afd49254:43181 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1
2024-12-06T05:31:25,336 DEBUG [RS:0;ec46afd49254:43181 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3
2024-12-06T05:31:25,336 DEBUG [RS:0;ec46afd49254:43181 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3
2024-12-06T05:31:25,337 INFO  [RS:0;ec46afd49254:43181 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:25,337 INFO  [RS:0;ec46afd49254:43181 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:25,337 INFO  [RS:0;ec46afd49254:43181 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:25,337 INFO  [RS:0;ec46afd49254:43181 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:25,337 INFO  [RS:0;ec46afd49254:43181 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:25,337 INFO  [RS:0;ec46afd49254:43181 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43181,1733463084850-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T05:31:25,352 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-06T05:31:25,352 INFO  [RS:0;ec46afd49254:43181 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43181,1733463084850-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:25,352 INFO  [RS:0;ec46afd49254:43181 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:25,352 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.Replication(171): ec46afd49254,43181,1733463084850 started
2024-12-06T05:31:25,365 INFO  [RS:0;ec46afd49254:43181 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:25,365 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.HRegionServer(1482): Serving as ec46afd49254,43181,1733463084850, RpcServer on ec46afd49254/172.17.0.3:43181, sessionid=0x101a91f8a200001
2024-12-06T05:31:25,365 DEBUG [RS:0;ec46afd49254:43181 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-06T05:31:25,365 DEBUG [RS:0;ec46afd49254:43181 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec46afd49254,43181,1733463084850
2024-12-06T05:31:25,365 DEBUG [RS:0;ec46afd49254:43181 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,43181,1733463084850'
2024-12-06T05:31:25,365 DEBUG [RS:0;ec46afd49254:43181 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-06T05:31:25,366 DEBUG [RS:0;ec46afd49254:43181 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-06T05:31:25,366 DEBUG [RS:0;ec46afd49254:43181 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-06T05:31:25,366 DEBUG [RS:0;ec46afd49254:43181 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-06T05:31:25,366 DEBUG [RS:0;ec46afd49254:43181 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec46afd49254,43181,1733463084850
2024-12-06T05:31:25,366 DEBUG [RS:0;ec46afd49254:43181 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,43181,1733463084850'
2024-12-06T05:31:25,366 DEBUG [RS:0;ec46afd49254:43181 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-06T05:31:25,367 DEBUG [RS:0;ec46afd49254:43181 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-06T05:31:25,367 DEBUG [RS:0;ec46afd49254:43181 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-06T05:31:25,367 INFO  [RS:0;ec46afd49254:43181 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-06T05:31:25,367 INFO  [RS:0;ec46afd49254:43181 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-06T05:31:25,389 WARN  [ec46afd49254:37087 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-06T05:31:25,469 INFO  [RS:0;ec46afd49254:43181 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C43181%2C1733463084850, suffix=, logDir=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/WALs/ec46afd49254,43181,1733463084850, archiveDir=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/oldWALs, maxLogs=32
2024-12-06T05:31:25,470 INFO  [RS:0;ec46afd49254:43181 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C43181%2C1733463084850.1733463085470
2024-12-06T05:31:25,478 INFO  [RS:0;ec46afd49254:43181 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/WALs/ec46afd49254,43181,1733463084850/ec46afd49254%2C43181%2C1733463084850.1733463085470
2024-12-06T05:31:25,479 DEBUG [RS:0;ec46afd49254:43181 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44163:44163),(127.0.0.1/127.0.0.1:41105:41105)]
2024-12-06T05:31:25,640 DEBUG [ec46afd49254:37087 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-06T05:31:25,640 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ec46afd49254,43181,1733463084850
2024-12-06T05:31:25,642 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec46afd49254,43181,1733463084850, state=OPENING
2024-12-06T05:31:25,647 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-06T05:31:25,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:25,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:25,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:31:25,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:31:25,658 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T05:31:25,658 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ec46afd49254,43181,1733463084850}]
2024-12-06T05:31:25,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:25,811 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-06T05:31:25,813 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56025, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-06T05:31:25,816 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-06T05:31:25,816 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:31:25,818 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C43181%2C1733463084850.meta, suffix=.meta, logDir=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/WALs/ec46afd49254,43181,1733463084850, archiveDir=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/oldWALs, maxLogs=32
2024-12-06T05:31:25,818 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ec46afd49254%2C43181%2C1733463084850.meta.1733463085818.meta
2024-12-06T05:31:25,828 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/WALs/ec46afd49254,43181,1733463084850/ec46afd49254%2C43181%2C1733463084850.meta.1733463085818.meta
2024-12-06T05:31:25,843 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44163:44163),(127.0.0.1/127.0.0.1:41105:41105)]
2024-12-06T05:31:25,848 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-06T05:31:25,848 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-06T05:31:25,849 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-06T05:31:25,849 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-06T05:31:25,849 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-06T05:31:25,849 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T05:31:25,849 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-06T05:31:25,849 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-06T05:31:25,850 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T05:31:25,851 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T05:31:25,851 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:31:25,851 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:31:25,851 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T05:31:25,852 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T05:31:25,852 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:31:25,852 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:31:25,852 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T05:31:25,853 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T05:31:25,853 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:31:25,853 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:31:25,854 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T05:31:25,854 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T05:31:25,854 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T05:31:25,855 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T05:31:25,855 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T05:31:25,855 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/data/hbase/meta/1588230740
2024-12-06T05:31:25,856 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/data/hbase/meta/1588230740
2024-12-06T05:31:25,857 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T05:31:25,857 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T05:31:25,857 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T05:31:25,858 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T05:31:25,859 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=759460, jitterRate=-0.03429713845252991}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T05:31:25,859 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-06T05:31:25,859 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733463085849Writing region info on filesystem at 1733463085849Initializing all the Stores at 1733463085850 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463085850Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463085850Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463085850Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463085850Cleaning up temporary data from old regions at 1733463085857 (+7 ms)Running coprocessor post-open hooks at 1733463085859 (+2 ms)Region opened successfully at 1733463085859
2024-12-06T05:31:25,860 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733463085810
2024-12-06T05:31:25,862 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-06T05:31:25,862 INFO  [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-06T05:31:25,863 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ec46afd49254,43181,1733463084850
2024-12-06T05:31:25,863 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec46afd49254,43181,1733463084850, state=OPEN
2024-12-06T05:31:25,969 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:26,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T05:31:26,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T05:31:26,037 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ec46afd49254,43181,1733463084850
2024-12-06T05:31:26,037 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:31:26,037 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T05:31:26,041 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-06T05:31:26,041 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec46afd49254,43181,1733463084850 in 379 msec
2024-12-06T05:31:26,045 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-06T05:31:26,045 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 803 msec
2024-12-06T05:31:26,046 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T05:31:26,046 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-06T05:31:26,048 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T05:31:26,048 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec46afd49254,43181,1733463084850, seqNum=-1]
2024-12-06T05:31:26,048 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T05:31:26,050 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60725, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T05:31:26,055 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 866 msec
2024-12-06T05:31:26,055 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733463086055, completionTime=-1
2024-12-06T05:31:26,055 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-06T05:31:26,056 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-06T05:31:26,058 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-06T05:31:26,058 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733463146058
2024-12-06T05:31:26,058 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733463206058
2024-12-06T05:31:26,058 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-06T05:31:26,058 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,37087,1733463084674-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:26,058 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,37087,1733463084674-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:26,058 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,37087,1733463084674-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:26,058 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ec46afd49254:37087, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:26,059 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:26,059 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:26,060 DEBUG [master/ec46afd49254:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T05:31:26,063 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.142sec
2024-12-06T05:31:26,063 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-06T05:31:26,063 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-06T05:31:26,063 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-06T05:31:26,063 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-06T05:31:26,063 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-06T05:31:26,064 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,37087,1733463084674-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T05:31:26,064 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,37087,1733463084674-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-06T05:31:26,066 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-06T05:31:26,066 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-06T05:31:26,066 INFO  [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,37087,1733463084674-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T05:31:26,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4457fb1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:31:26,079 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ec46afd49254,37087,-1 for getting cluster id
2024-12-06T05:31:26,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-06T05:31:26,080 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20172045-abb2-4205-82b9-e5db37da1c00'
2024-12-06T05:31:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-06T05:31:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20172045-abb2-4205-82b9-e5db37da1c00"

2024-12-06T05:31:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f0d5326, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:31:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ec46afd49254,37087,-1]
2024-12-06T05:31:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-06T05:31:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:31:26,083 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-06T05:31:26,084 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ce49b8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T05:31:26,084 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T05:31:26,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec46afd49254,43181,1733463084850, seqNum=-1]
2024-12-06T05:31:26,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T05:31:26,087 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T05:31:26,093 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ec46afd49254,37087,1733463084674
2024-12-06T05:31:26,093 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T05:31:26,096 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-06T05:31:26,097 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T05:31:26,099 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/WALs/test.com,8080,1, archiveDir=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/oldWALs, maxLogs=32
2024-12-06T05:31:26,099 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733463086099
2024-12-06T05:31:26,104 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/WALs/test.com,8080,1/test.com%2C8080%2C1.1733463086099
2024-12-06T05:31:26,108 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41105:41105),(127.0.0.1/127.0.0.1:44163:44163)]
2024-12-06T05:31:26,109 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733463086109
2024-12-06T05:31:26,113 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,114 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,114 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,114 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,114 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,114 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/WALs/test.com,8080,1/test.com%2C8080%2C1.1733463086099 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/WALs/test.com,8080,1/test.com%2C8080%2C1.1733463086109
2024-12-06T05:31:26,115 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44163:44163),(127.0.0.1/127.0.0.1:41105:41105)]
2024-12-06T05:31:26,115 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/WALs/test.com,8080,1/test.com%2C8080%2C1.1733463086099 is not closed yet, will try archiving it next time
2024-12-06T05:31:26,115 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,115 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,115 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,115 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,115 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741835_1011 (size=93)
2024-12-06T05:31:26,115 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,115 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741835_1011 (size=93)
2024-12-06T05:31:26,116 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/WALs/test.com,8080,1/test.com%2C8080%2C1.1733463086099 to hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/oldWALs/test.com%2C8080%2C1.1733463086099
2024-12-06T05:31:26,117 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741836_1012 (size=93)
2024-12-06T05:31:26,117 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741836_1012 (size=93)
2024-12-06T05:31:26,505 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-06T05:31:26,521 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/oldWALs
2024-12-06T05:31:26,521 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1733463086109)
2024-12-06T05:31:26,521 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-06T05:31:26,521 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T05:31:26,522 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:31:26,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:31:26,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:31:26,522 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-06T05:31:26,522 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-06T05:31:26,522 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=236212020, stopped=false
2024-12-06T05:31:26,522 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ec46afd49254,37087,1733463084674
2024-12-06T05:31:26,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T05:31:26,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T05:31:26,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:26,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:26,574 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T05:31:26,574 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T05:31:26,574 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:31:26,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:31:26,575 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:31:26,575 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ec46afd49254,43181,1733463084850' *****
2024-12-06T05:31:26,575 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-06T05:31:26,575 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T05:31:26,575 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-06T05:31:26,575 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-06T05:31:26,575 INFO  [RS:0;ec46afd49254:43181 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-06T05:31:26,576 INFO  [RS:0;ec46afd49254:43181 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-06T05:31:26,576 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.HRegionServer(959): stopping server ec46afd49254,43181,1733463084850
2024-12-06T05:31:26,576 INFO  [RS:0;ec46afd49254:43181 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T05:31:26,576 INFO  [RS:0;ec46afd49254:43181 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ec46afd49254:43181.
2024-12-06T05:31:26,576 DEBUG [RS:0;ec46afd49254:43181 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T05:31:26,576 DEBUG [RS:0;ec46afd49254:43181 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:31:26,576 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-06T05:31:26,576 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-06T05:31:26,576 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-06T05:31:26,576 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-06T05:31:26,577 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-06T05:31:26,577 DEBUG [RS:0;ec46afd49254:43181 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-06T05:31:26,577 DEBUG [RS:0;ec46afd49254:43181 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-06T05:31:26,577 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T05:31:26,577 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T05:31:26,577 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T05:31:26,577 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T05:31:26,577 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T05:31:26,577 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-12-06T05:31:26,599 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/data/hbase/meta/1588230740/.tmp/ns/23abc24adc7c451c8af66b05914ac28b is 43, key is default/ns:d/1733463086050/Put/seqid=0
2024-12-06T05:31:26,603 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741837_1013 (size=5153)
2024-12-06T05:31:26,603 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741837_1013 (size=5153)
2024-12-06T05:31:26,603 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/data/hbase/meta/1588230740/.tmp/ns/23abc24adc7c451c8af66b05914ac28b
2024-12-06T05:31:26,608 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/data/hbase/meta/1588230740/.tmp/ns/23abc24adc7c451c8af66b05914ac28b as hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/data/hbase/meta/1588230740/ns/23abc24adc7c451c8af66b05914ac28b
2024-12-06T05:31:26,613 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/data/hbase/meta/1588230740/ns/23abc24adc7c451c8af66b05914ac28b, entries=2, sequenceid=6, filesize=5.0 K
2024-12-06T05:31:26,614 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 37ms, sequenceid=6, compaction requested=false
2024-12-06T05:31:26,618 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-06T05:31:26,619 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T05:31:26,619 INFO  [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T05:31:26,619 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733463086577Running coprocessor pre-close hooks at 1733463086577Disabling compacts and flushes for region at 1733463086577Disabling writes for close at 1733463086577Obtaining lock to block concurrent updates at 1733463086577Preparing flush snapshotting stores in 1588230740 at 1733463086577Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733463086578 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733463086579 (+1 ms)Flushing 1588230740/ns: creating writer at 1733463086579Flushing 1588230740/ns: appending metadata at 1733463086598 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1733463086598Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@488ab077: reopening flushed file at 1733463086607 (+9 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 37ms, sequenceid=6, compaction requested=false at 1733463086614 (+7 ms)Writing region close event to WAL at 1733463086615 (+1 ms)Running coprocessor post-close hooks at 1733463086619 (+4 ms)Closed at 1733463086619
2024-12-06T05:31:26,619 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-06T05:31:26,777 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.HRegionServer(976): stopping server ec46afd49254,43181,1733463084850; all regions closed.
2024-12-06T05:31:26,778 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,778 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,778 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,778 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,778 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741834_1010 (size=1152)
2024-12-06T05:31:26,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741834_1010 (size=1152)
2024-12-06T05:31:26,783 DEBUG [RS:0;ec46afd49254:43181 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/oldWALs
2024-12-06T05:31:26,783 INFO  [RS:0;ec46afd49254:43181 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ec46afd49254%2C43181%2C1733463084850.meta:.meta(num 1733463085818)
2024-12-06T05:31:26,783 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,783 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,783 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,783 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,783 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741833_1009 (size=93)
2024-12-06T05:31:26,786 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741833_1009 (size=93)
2024-12-06T05:31:26,788 DEBUG [RS:0;ec46afd49254:43181 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/oldWALs
2024-12-06T05:31:26,788 INFO  [RS:0;ec46afd49254:43181 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ec46afd49254%2C43181%2C1733463084850:(num 1733463085470)
2024-12-06T05:31:26,788 DEBUG [RS:0;ec46afd49254:43181 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T05:31:26,788 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T05:31:26,788 INFO  [RS:0;ec46afd49254:43181 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T05:31:26,788 INFO  [RS:0;ec46afd49254:43181 {}] hbase.ChoreService(370): Chore service for: regionserver/ec46afd49254:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-06T05:31:26,788 INFO  [RS:0;ec46afd49254:43181 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T05:31:26,788 INFO  [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T05:31:26,789 INFO  [RS:0;ec46afd49254:43181 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43181
2024-12-06T05:31:26,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T05:31:26,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec46afd49254,43181,1733463084850
2024-12-06T05:31:26,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,34771,1733462892328/ec46afd49254%2C34771%2C1733462892328.1733462892575
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:26,795 INFO  [RS:0;ec46afd49254:43181 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T05:31:26,805 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec46afd49254,43181,1733463084850]
2024-12-06T05:31:26,815 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ec46afd49254,43181,1733463084850 already deleted, retry=false
2024-12-06T05:31:26,816 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ec46afd49254,43181,1733463084850 expired; onlineServers=0
2024-12-06T05:31:26,816 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ec46afd49254,37087,1733463084674' *****
2024-12-06T05:31:26,816 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-06T05:31:26,816 INFO  [M:0;ec46afd49254:37087 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T05:31:26,816 INFO  [M:0;ec46afd49254:37087 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T05:31:26,816 DEBUG [M:0;ec46afd49254:37087 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-06T05:31:26,816 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-06T05:31:26,816 DEBUG [M:0;ec46afd49254:37087 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-06T05:31:26,816 DEBUG [master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733463085194 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733463085194,5,FailOnTimeoutGroup]
2024-12-06T05:31:26,816 DEBUG [master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733463085194 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733463085194,5,FailOnTimeoutGroup]
2024-12-06T05:31:26,816 INFO  [M:0;ec46afd49254:37087 {}] hbase.ChoreService(370): Chore service for: master/ec46afd49254:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-06T05:31:26,816 INFO  [M:0;ec46afd49254:37087 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T05:31:26,816 DEBUG [M:0;ec46afd49254:37087 {}] master.HMaster(1795): Stopping service threads
2024-12-06T05:31:26,816 INFO  [M:0;ec46afd49254:37087 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-06T05:31:26,816 INFO  [M:0;ec46afd49254:37087 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T05:31:26,816 INFO  [M:0;ec46afd49254:37087 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-06T05:31:26,816 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-06T05:31:26,826 DEBUG [M:0;ec46afd49254:37087 {}] zookeeper.ZKUtil(347): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-06T05:31:26,826 WARN  [M:0;ec46afd49254:37087 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-06T05:31:26,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-06T05:31:26,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T05:31:26,827 INFO  [M:0;ec46afd49254:37087 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/.lastflushedseqids
2024-12-06T05:31:26,832 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741838_1014 (size=99)
2024-12-06T05:31:26,832 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741838_1014 (size=99)
2024-12-06T05:31:26,833 INFO  [M:0;ec46afd49254:37087 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-06T05:31:26,833 INFO  [M:0;ec46afd49254:37087 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-06T05:31:26,833 DEBUG [M:0;ec46afd49254:37087 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T05:31:26,833 INFO  [M:0;ec46afd49254:37087 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:31:26,833 DEBUG [M:0;ec46afd49254:37087 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:31:26,833 DEBUG [M:0;ec46afd49254:37087 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T05:31:26,833 DEBUG [M:0;ec46afd49254:37087 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:31:26,833 INFO  [M:0;ec46afd49254:37087 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-12-06T05:31:26,837 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,837 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,837 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,837 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,838 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,838 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,838 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,838 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,838 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,839 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,852 DEBUG [M:0;ec46afd49254:37087 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7b9ceb1348664585bfe01f6ff573931a is 82, key is hbase:meta,,1/info:regioninfo/1733463085862/Put/seqid=0
2024-12-06T05:31:26,856 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,857 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,857 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,857 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,857 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,857 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,860 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,860 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,860 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,861 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741839_1015 (size=5672)
2024-12-06T05:31:26,861 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741839_1015 (size=5672)
2024-12-06T05:31:26,863 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T05:31:26,869 INFO  [M:0;ec46afd49254:37087 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7b9ceb1348664585bfe01f6ff573931a
2024-12-06T05:31:26,888 DEBUG [M:0;ec46afd49254:37087 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c54fe4ebc1ac4b14b593a6cd6053dc03 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733463086055/Put/seqid=0
2024-12-06T05:31:26,892 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741840_1016 (size=5275)
2024-12-06T05:31:26,892 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741840_1016 (size=5275)
2024-12-06T05:31:26,893 INFO  [M:0;ec46afd49254:37087 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c54fe4ebc1ac4b14b593a6cd6053dc03
2024-12-06T05:31:26,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:31:26,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43181-0x101a91f8a200001, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:31:26,905 INFO  [RS:0;ec46afd49254:43181 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T05:31:26,905 INFO  [RS:0;ec46afd49254:43181 {}] regionserver.HRegionServer(1031): Exiting; stopping=ec46afd49254,43181,1733463084850; zookeeper connection closed.
2024-12-06T05:31:26,906 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7911083c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7911083c
2024-12-06T05:31:26,907 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-06T05:31:26,915 DEBUG [M:0;ec46afd49254:37087 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/848035bbe1bd43cfb73595c1db3aee50 is 69, key is ec46afd49254,43181,1733463084850/rs:state/1733463085317/Put/seqid=0
2024-12-06T05:31:26,919 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741841_1017 (size=5156)
2024-12-06T05:31:26,920 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741841_1017 (size=5156)
2024-12-06T05:31:26,920 INFO  [M:0;ec46afd49254:37087 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/848035bbe1bd43cfb73595c1db3aee50
2024-12-06T05:31:26,938 DEBUG [M:0;ec46afd49254:37087 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f731708db7344b099d7108dc29707cbb is 52, key is load_balancer_on/state:d/1733463086095/Put/seqid=0
2024-12-06T05:31:26,942 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741842_1018 (size=5056)
2024-12-06T05:31:26,943 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741842_1018 (size=5056)
2024-12-06T05:31:26,943 INFO  [M:0;ec46afd49254:37087 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f731708db7344b099d7108dc29707cbb
2024-12-06T05:31:26,948 DEBUG [M:0;ec46afd49254:37087 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7b9ceb1348664585bfe01f6ff573931a as hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7b9ceb1348664585bfe01f6ff573931a
2024-12-06T05:31:26,953 INFO  [M:0;ec46afd49254:37087 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7b9ceb1348664585bfe01f6ff573931a, entries=8, sequenceid=29, filesize=5.5 K
2024-12-06T05:31:26,954 DEBUG [M:0;ec46afd49254:37087 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c54fe4ebc1ac4b14b593a6cd6053dc03 as hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c54fe4ebc1ac4b14b593a6cd6053dc03
2024-12-06T05:31:26,958 INFO  [M:0;ec46afd49254:37087 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c54fe4ebc1ac4b14b593a6cd6053dc03, entries=3, sequenceid=29, filesize=5.2 K
2024-12-06T05:31:26,959 DEBUG [M:0;ec46afd49254:37087 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/848035bbe1bd43cfb73595c1db3aee50 as hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/848035bbe1bd43cfb73595c1db3aee50
2024-12-06T05:31:26,964 INFO  [M:0;ec46afd49254:37087 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/848035bbe1bd43cfb73595c1db3aee50, entries=1, sequenceid=29, filesize=5.0 K
2024-12-06T05:31:26,965 DEBUG [M:0;ec46afd49254:37087 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f731708db7344b099d7108dc29707cbb as hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f731708db7344b099d7108dc29707cbb
2024-12-06T05:31:26,970 INFO  [M:0;ec46afd49254:37087 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/b8ef2495-b2c0-264b-1f8e-82d9dbb6ee33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f731708db7344b099d7108dc29707cbb, entries=1, sequenceid=29, filesize=4.9 K
2024-12-06T05:31:26,969 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41215/user/jenkins/test-data/8f8306eb-83e7-568b-f568-879d07ff2086/WALs/ec46afd49254,39957,1733462890475/ec46afd49254%2C39957%2C1733462890475.meta.1733462892145.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T05:31:26,970 INFO  [M:0;ec46afd49254:37087 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 137ms, sequenceid=29, compaction requested=false
2024-12-06T05:31:26,972 INFO  [M:0;ec46afd49254:37087 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T05:31:26,972 DEBUG [M:0;ec46afd49254:37087 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733463086833Disabling compacts and flushes for region at 1733463086833Disabling writes for close at 1733463086833Obtaining lock to block concurrent updates at 1733463086833Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733463086833Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733463086834 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733463086834Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733463086834Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733463086852 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733463086852Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733463086873 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733463086887 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733463086887Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733463086898 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733463086915 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733463086915Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733463086924 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733463086938 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733463086938Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37b17e: reopening flushed file at 1733463086947 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ed589bb: reopening flushed file at 1733463086953 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@490798a8: reopening flushed file at 1733463086958 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f219741: reopening flushed file at 1733463086964 (+6 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 137ms, sequenceid=29, compaction requested=false at 1733463086970 (+6 ms)Writing region close event to WAL at 1733463086972 (+2 ms)Closed at 1733463086972
2024-12-06T05:31:26,973 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,973 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,973 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,973 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,973 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T05:31:26,975 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46395 is added to blk_1073741830_1006 (size=10311)
2024-12-06T05:31:26,975 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37533 is added to blk_1073741830_1006 (size=10311)
2024-12-06T05:31:26,976 INFO  [M:0;ec46afd49254:37087 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-06T05:31:26,976 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T05:31:26,976 INFO  [M:0;ec46afd49254:37087 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37087
2024-12-06T05:31:26,976 INFO  [M:0;ec46afd49254:37087 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T05:31:27,084 INFO  [M:0;ec46afd49254:37087 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T05:31:27,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:31:27,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37087-0x101a91f8a200000, quorum=127.0.0.1:56790, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T05:31:27,089 DEBUG [master/ec46afd49254:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=9, created chunk count=9, reused chunk count=78, reuseRatio=89.66%
2024-12-06T05:31:27,089 DEBUG [master/ec46afd49254:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-12-06T05:31:27,164 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@627a202d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:31:27,165 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22ed154c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:31:27,165 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:31:27,165 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6bfe0bbd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:31:27,165 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@49e6dd92{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/hadoop.log.dir/,STOPPED}
2024-12-06T05:31:27,166 WARN  [BP-798507793-172.17.0.3-1733463082470 heartbeating to localhost/127.0.0.1:40171 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:31:27,166 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:31:27,166 WARN  [BP-798507793-172.17.0.3-1733463082470 heartbeating to localhost/127.0.0.1:40171 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-798507793-172.17.0.3-1733463082470 (Datanode Uuid 35e86b08-28a6-4ae7-8b5f-4cf23ca5da70) service to localhost/127.0.0.1:40171
2024-12-06T05:31:27,166 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:31:27,166 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/cluster_1f6d5f9e-3a16-da76-5706-ef7c8d453a05/data/data3/current/BP-798507793-172.17.0.3-1733463082470 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:31:27,167 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/cluster_1f6d5f9e-3a16-da76-5706-ef7c8d453a05/data/data4/current/BP-798507793-172.17.0.3-1733463082470 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:31:27,167 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:31:27,173 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@9fc2daa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T05:31:27,173 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2a1af98b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:31:27,173 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:31:27,173 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1040cecb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:31:27,173 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fcd61c6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/hadoop.log.dir/,STOPPED}
2024-12-06T05:31:27,175 WARN  [BP-798507793-172.17.0.3-1733463082470 heartbeating to localhost/127.0.0.1:40171 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T05:31:27,175 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T05:31:27,175 WARN  [BP-798507793-172.17.0.3-1733463082470 heartbeating to localhost/127.0.0.1:40171 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-798507793-172.17.0.3-1733463082470 (Datanode Uuid a5b3a820-c3bb-498c-a6ae-4a86bd48eae9) service to localhost/127.0.0.1:40171
2024-12-06T05:31:27,175 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T05:31:27,175 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/cluster_1f6d5f9e-3a16-da76-5706-ef7c8d453a05/data/data1/current/BP-798507793-172.17.0.3-1733463082470 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:31:27,175 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/cluster_1f6d5f9e-3a16-da76-5706-ef7c8d453a05/data/data2/current/BP-798507793-172.17.0.3-1733463082470 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T05:31:27,176 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T05:31:27,181 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@61edd007{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T05:31:27,181 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7d664f93{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T05:31:27,181 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T05:31:27,182 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4c66b7d1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T05:31:27,182 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b72d363{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93ec1247-4bb9-0822-7142-ffb7efb059cb/hadoop.log.dir/,STOPPED}
2024-12-06T05:31:27,187 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-06T05:31:27,202 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-06T05:31:27,210 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=268 (was 232)
Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40171 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40171
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40171 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40171
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:40171
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40171 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:40171
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40171
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=532 (was 503) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=259 (was 259), ProcessCount=11 (was 11), AvailableMemoryMB=6299 (was 6311)