2024-11-19 00:05:21,691 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-19 00:05:21,706 main DEBUG Took 0.013413 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-19 00:05:21,707 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-19 00:05:21,707 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-19 00:05:21,708 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-19 00:05:21,710 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 00:05:21,717 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-19 00:05:21,732 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 00:05:21,734 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 00:05:21,735 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 00:05:21,735 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 00:05:21,736 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 00:05:21,736 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 00:05:21,737 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 00:05:21,738 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 00:05:21,738 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 00:05:21,739 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 00:05:21,740 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 00:05:21,740 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 00:05:21,741 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 00:05:21,741 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 00:05:21,742 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 00:05:21,742 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 00:05:21,743 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 00:05:21,743 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 00:05:21,744 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 00:05:21,744 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 00:05:21,745 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 00:05:21,745 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 00:05:21,746 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 00:05:21,746 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 00:05:21,747 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 00:05:21,747 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-19 00:05:21,749 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 00:05:21,751 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-19 00:05:21,753 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-19 00:05:21,754 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-19 00:05:21,755 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-19 00:05:21,755 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-19 00:05:21,766 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-19 00:05:21,768 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-19 00:05:21,770 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-19 00:05:21,771 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-19 00:05:21,771 main DEBUG createAppenders(={Console})
2024-11-19 00:05:21,772 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-11-19 00:05:21,772 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-19 00:05:21,773 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-11-19 00:05:21,773 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-19 00:05:21,774 main DEBUG OutputStream closed
2024-11-19 00:05:21,774 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-19 00:05:21,775 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-19 00:05:21,775 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-11-19 00:05:21,856 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-19 00:05:21,858 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-19 00:05:21,859 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-19 00:05:21,859 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-19 00:05:21,860 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-19 00:05:21,860 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-19 00:05:21,860 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-19 00:05:21,861 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-19 00:05:21,861 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-19 00:05:21,861 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-19 00:05:21,862 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-19 00:05:21,862 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-19 00:05:21,862 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-19 00:05:21,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-19 00:05:21,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-19 00:05:21,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-19 00:05:21,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-19 00:05:21,864 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-19 00:05:21,866 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-19 00:05:21,866 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-11-19 00:05:21,867 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-19 00:05:21,867 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-11-19T00:05:22,092 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9
2024-11-19 00:05:22,095 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-19 00:05:22,096 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-19T00:05:22,105 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-11-19T00:05:22,138 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=166, ProcessCount=11, AvailableMemoryMB=5141
2024-11-19T00:05:22,141 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-19T00:05:22,155 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/cluster_49ed2eb7-57e5-c679-c6b0-21c3fdb477d0, deleteOnExit=true
2024-11-19T00:05:22,156 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-19T00:05:22,157 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/test.cache.data in system properties and HBase conf
2024-11-19T00:05:22,157 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/hadoop.tmp.dir in system properties and HBase conf
2024-11-19T00:05:22,158 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/hadoop.log.dir in system properties and HBase conf
2024-11-19T00:05:22,158 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-19T00:05:22,159 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-19T00:05:22,159 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-19T00:05:22,245 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-19T00:05:22,333 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-19T00:05:22,337 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-19T00:05:22,337 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-19T00:05:22,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-19T00:05:22,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T00:05:22,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-19T00:05:22,339 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-19T00:05:22,339 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T00:05:22,340 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T00:05:22,340 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-19T00:05:22,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/nfs.dump.dir in system properties and HBase conf
2024-11-19T00:05:22,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/java.io.tmpdir in system properties and HBase conf
2024-11-19T00:05:22,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T00:05:22,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-19T00:05:22,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-19T00:05:22,829 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T00:05:23,178 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-19T00:05:23,269 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2275ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-19T00:05:23,357 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:05:23,428 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:05:23,448 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:05:23,448 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:05:23,449 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T00:05:23,463 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:05:23,465 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:05:23,467 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:05:23,674 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/java.io.tmpdir/jetty-localhost-36199-hadoop-hdfs-3_4_1-tests_jar-_-any-16378338215426280053/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T00:05:23,681 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:36199}
2024-11-19T00:05:23,681 INFO  [Time-limited test {}] server.Server(415): Started @2689ms
2024-11-19T00:05:23,710 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T00:05:24,063 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:05:24,070 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:05:24,072 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:05:24,072 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:05:24,072 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T00:05:24,073 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:05:24,073 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:05:24,194 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/java.io.tmpdir/jetty-localhost-34435-hadoop-hdfs-3_4_1-tests_jar-_-any-431676792983416553/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:05:24,195 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:34435}
2024-11-19T00:05:24,195 INFO  [Time-limited test {}] server.Server(415): Started @3202ms
2024-11-19T00:05:24,253 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:05:24,381 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:05:24,386 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:05:24,387 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:05:24,388 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:05:24,388 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T00:05:24,389 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:05:24,389 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:05:24,509 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/java.io.tmpdir/jetty-localhost-40393-hadoop-hdfs-3_4_1-tests_jar-_-any-15957565494798742086/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:05:24,510 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:40393}
2024-11-19T00:05:24,511 INFO  [Time-limited test {}] server.Server(415): Started @3518ms
2024-11-19T00:05:24,513 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:05:24,705 WARN  [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/cluster_49ed2eb7-57e5-c679-c6b0-21c3fdb477d0/data/data3/current/BP-1539334133-172.17.0.2-1731974722923/current, will proceed with Du for space computation calculation, 
2024-11-19T00:05:24,705 WARN  [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/cluster_49ed2eb7-57e5-c679-c6b0-21c3fdb477d0/data/data2/current/BP-1539334133-172.17.0.2-1731974722923/current, will proceed with Du for space computation calculation, 
2024-11-19T00:05:24,705 WARN  [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/cluster_49ed2eb7-57e5-c679-c6b0-21c3fdb477d0/data/data1/current/BP-1539334133-172.17.0.2-1731974722923/current, will proceed with Du for space computation calculation, 
2024-11-19T00:05:24,705 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/cluster_49ed2eb7-57e5-c679-c6b0-21c3fdb477d0/data/data4/current/BP-1539334133-172.17.0.2-1731974722923/current, will proceed with Du for space computation calculation, 
2024-11-19T00:05:24,764 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:05:24,765 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:05:24,839 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6f6fde160de62dc2 with lease ID 0x22fd14c3ef85b46a: Processing first storage report for DS-0229ef1f-3c2a-4337-8eb3-f1693e496632 from datanode DatanodeRegistration(127.0.0.1:40913, datanodeUuid=3fb28dbf-0f80-4a88-86f4-b16ed1f50c31, infoPort=37793, infoSecurePort=0, ipcPort=36849, storageInfo=lv=-57;cid=testClusterID;nsid=1854955811;c=1731974722923)
2024-11-19T00:05:24,842 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6f6fde160de62dc2 with lease ID 0x22fd14c3ef85b46a: from storage DS-0229ef1f-3c2a-4337-8eb3-f1693e496632 node DatanodeRegistration(127.0.0.1:40913, datanodeUuid=3fb28dbf-0f80-4a88-86f4-b16ed1f50c31, infoPort=37793, infoSecurePort=0, ipcPort=36849, storageInfo=lv=-57;cid=testClusterID;nsid=1854955811;c=1731974722923), blocks: 0, hasStaleStorage: true, processing time: 4 msecs, invalidatedBlocks: 0
2024-11-19T00:05:24,843 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7e0e7c93bc3f1af1 with lease ID 0x22fd14c3ef85b46b: Processing first storage report for DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32 from datanode DatanodeRegistration(127.0.0.1:43229, datanodeUuid=30a8a02c-31c3-4278-8f24-cc4e0f6abb61, infoPort=42447, infoSecurePort=0, ipcPort=40871, storageInfo=lv=-57;cid=testClusterID;nsid=1854955811;c=1731974722923)
2024-11-19T00:05:24,843 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7e0e7c93bc3f1af1 with lease ID 0x22fd14c3ef85b46b: from storage DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32 node DatanodeRegistration(127.0.0.1:43229, datanodeUuid=30a8a02c-31c3-4278-8f24-cc4e0f6abb61, infoPort=42447, infoSecurePort=0, ipcPort=40871, storageInfo=lv=-57;cid=testClusterID;nsid=1854955811;c=1731974722923), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:05:24,844 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6f6fde160de62dc2 with lease ID 0x22fd14c3ef85b46a: Processing first storage report for DS-58275a4a-8ddd-4d32-9364-72d67878fe22 from datanode DatanodeRegistration(127.0.0.1:40913, datanodeUuid=3fb28dbf-0f80-4a88-86f4-b16ed1f50c31, infoPort=37793, infoSecurePort=0, ipcPort=36849, storageInfo=lv=-57;cid=testClusterID;nsid=1854955811;c=1731974722923)
2024-11-19T00:05:24,844 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6f6fde160de62dc2 with lease ID 0x22fd14c3ef85b46a: from storage DS-58275a4a-8ddd-4d32-9364-72d67878fe22 node DatanodeRegistration(127.0.0.1:40913, datanodeUuid=3fb28dbf-0f80-4a88-86f4-b16ed1f50c31, infoPort=37793, infoSecurePort=0, ipcPort=36849, storageInfo=lv=-57;cid=testClusterID;nsid=1854955811;c=1731974722923), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-19T00:05:24,844 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7e0e7c93bc3f1af1 with lease ID 0x22fd14c3ef85b46b: Processing first storage report for DS-6bab6d5b-0c45-4d72-905f-6fc08303f7b0 from datanode DatanodeRegistration(127.0.0.1:43229, datanodeUuid=30a8a02c-31c3-4278-8f24-cc4e0f6abb61, infoPort=42447, infoSecurePort=0, ipcPort=40871, storageInfo=lv=-57;cid=testClusterID;nsid=1854955811;c=1731974722923)
2024-11-19T00:05:24,844 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7e0e7c93bc3f1af1 with lease ID 0x22fd14c3ef85b46b: from storage DS-6bab6d5b-0c45-4d72-905f-6fc08303f7b0 node DatanodeRegistration(127.0.0.1:43229, datanodeUuid=30a8a02c-31c3-4278-8f24-cc4e0f6abb61, infoPort=42447, infoSecurePort=0, ipcPort=40871, storageInfo=lv=-57;cid=testClusterID;nsid=1854955811;c=1731974722923), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:05:24,917 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9
2024-11-19T00:05:24,994 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/cluster_49ed2eb7-57e5-c679-c6b0-21c3fdb477d0/zookeeper_0, clientPort=54464, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/cluster_49ed2eb7-57e5-c679-c6b0-21c3fdb477d0/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/cluster_49ed2eb7-57e5-c679-c6b0-21c3fdb477d0/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-19T00:05:25,004 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54464
2024-11-19T00:05:25,015 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:05:25,018 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:05:25,267 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741825_1001 (size=7)
2024-11-19T00:05:25,268 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741825_1001 (size=7)
2024-11-19T00:05:25,676 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4 with version=8
2024-11-19T00:05:25,676 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/hbase-staging
2024-11-19T00:05:25,769 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-19T00:05:26,024 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/9c732c6543ff:0 server-side Connection retries=45
2024-11-19T00:05:26,036 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:05:26,036 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T00:05:26,041 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T00:05:26,042 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:05:26,042 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T00:05:26,195 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-19T00:05:26,256 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-19T00:05:26,265 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-19T00:05:26,269 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T00:05:26,296 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 25018 (auto-detected)
2024-11-19T00:05:26,297 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-11-19T00:05:26,317 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45857
2024-11-19T00:05:26,338 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45857 connecting to ZooKeeper ensemble=127.0.0.1:54464
2024-11-19T00:05:26,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:458570x0, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T00:05:26,373 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45857-0x101084ff21a0000 connected
2024-11-19T00:05:26,400 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:05:26,402 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:05:26,412 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:05:26,416 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4, hbase.cluster.distributed=false
2024-11-19T00:05:26,439 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T00:05:26,444 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45857
2024-11-19T00:05:26,445 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45857
2024-11-19T00:05:26,446 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45857
2024-11-19T00:05:26,446 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45857
2024-11-19T00:05:26,449 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45857
2024-11-19T00:05:26,566 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/9c732c6543ff:0 server-side Connection retries=45
2024-11-19T00:05:26,568 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:05:26,568 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T00:05:26,568 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T00:05:26,568 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:05:26,568 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T00:05:26,571 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-19T00:05:26,574 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T00:05:26,575 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40793
2024-11-19T00:05:26,577 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40793 connecting to ZooKeeper ensemble=127.0.0.1:54464
2024-11-19T00:05:26,578 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:05:26,582 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:05:26,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:407930x0, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T00:05:26,590 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:05:26,590 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40793-0x101084ff21a0001 connected
2024-11-19T00:05:26,595 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-19T00:05:26,606 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-19T00:05:26,608 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-19T00:05:26,613 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T00:05:26,614 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40793
2024-11-19T00:05:26,614 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40793
2024-11-19T00:05:26,615 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40793
2024-11-19T00:05:26,617 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40793
2024-11-19T00:05:26,617 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40793
2024-11-19T00:05:26,636 DEBUG [M:0;9c732c6543ff:45857 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9c732c6543ff:45857
2024-11-19T00:05:26,637 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9c732c6543ff,45857,1731974725823
2024-11-19T00:05:26,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:05:26,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:05:26,647 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9c732c6543ff,45857,1731974725823
2024-11-19T00:05:26,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-19T00:05:26,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:05:26,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:05:26,671 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-19T00:05:26,672 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9c732c6543ff,45857,1731974725823 from backup master directory
2024-11-19T00:05:26,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:05:26,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9c732c6543ff,45857,1731974725823
2024-11-19T00:05:26,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:05:26,675 WARN  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T00:05:26,676 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9c732c6543ff,45857,1731974725823
2024-11-19T00:05:26,678 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-19T00:05:26,679 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-19T00:05:26,735 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/hbase.id] with ID: dc3b74e2-0fe7-4757-ba99-c0b2b65aae11
2024-11-19T00:05:26,736 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/.tmp/hbase.id
2024-11-19T00:05:26,749 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741826_1002 (size=42)
2024-11-19T00:05:26,749 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741826_1002 (size=42)
2024-11-19T00:05:26,750 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/.tmp/hbase.id]:[hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/hbase.id]
2024-11-19T00:05:26,796 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:05:26,801 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-19T00:05:26,820 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms.
2024-11-19T00:05:26,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:05:26,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:05:26,840 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741827_1003 (size=196)
2024-11-19T00:05:26,841 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741827_1003 (size=196)
2024-11-19T00:05:26,857 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T00:05:26,859 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-19T00:05:26,865 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:05:26,897 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741828_1004 (size=1189)
2024-11-19T00:05:26,897 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741828_1004 (size=1189)
2024-11-19T00:05:26,916 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store
2024-11-19T00:05:26,936 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741829_1005 (size=34)
2024-11-19T00:05:26,937 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741829_1005 (size=34)
2024-11-19T00:05:26,942 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-19T00:05:26,946 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:05:26,948 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T00:05:26,948 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:05:26,948 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:05:26,950 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T00:05:26,950 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:05:26,950 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:05:26,952 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731974726948Disabling compacts and flushes for region at 1731974726948Disabling writes for close at 1731974726950 (+2 ms)Writing region close event to WAL at 1731974726950Closed at 1731974726950
2024-11-19T00:05:26,954 WARN  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/.initializing
2024-11-19T00:05:26,954 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/WALs/9c732c6543ff,45857,1731974725823
2024-11-19T00:05:26,979 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C45857%2C1731974725823, suffix=, logDir=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/WALs/9c732c6543ff,45857,1731974725823, archiveDir=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/oldWALs, maxLogs=10
2024-11-19T00:05:26,991 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C45857%2C1731974725823.1731974726985
2024-11-19T00:05:27,014 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/WALs/9c732c6543ff,45857,1731974725823/9c732c6543ff%2C45857%2C1731974725823.1731974726985
2024-11-19T00:05:27,025 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37793:37793),(127.0.0.1/127.0.0.1:42447:42447)]
2024-11-19T00:05:27,027 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:05:27,027 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:05:27,031 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:05:27,032 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:05:27,069 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:05:27,095 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-19T00:05:27,099 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:05:27,102 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:05:27,103 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:05:27,106 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-19T00:05:27,106 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:05:27,107 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:05:27,107 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:05:27,110 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-19T00:05:27,110 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:05:27,111 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:05:27,111 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:05:27,114 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-19T00:05:27,114 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:05:27,115 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:05:27,115 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:05:27,119 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:05:27,120 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:05:27,126 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:05:27,126 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:05:27,129 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-19T00:05:27,134 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:05:27,138 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:05:27,139 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=863965, jitterRate=0.09858867526054382}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-19T00:05:27,146 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731974727044Initializing all the Stores at 1731974727046 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974727047 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974727047Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974727047Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974727047Cleaning up temporary data from old regions at 1731974727126 (+79 ms)Region opened successfully at 1731974727145 (+19 ms)
2024-11-19T00:05:27,147 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-19T00:05:27,186 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38bc7ce3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0
2024-11-19T00:05:27,219 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-19T00:05:27,230 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-19T00:05:27,231 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-19T00:05:27,234 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-19T00:05:27,235 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-19T00:05:27,241 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-19T00:05:27,241 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-19T00:05:27,267 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-19T00:05:27,276 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-19T00:05:27,279 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-19T00:05:27,281 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-19T00:05:27,283 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-19T00:05:27,285 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-19T00:05:27,287 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-19T00:05:27,290 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-19T00:05:27,292 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-19T00:05:27,293 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-19T00:05:27,295 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-19T00:05:27,312 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-19T00:05:27,314 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-19T00:05:27,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T00:05:27,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T00:05:27,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:05:27,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:05:27,321 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9c732c6543ff,45857,1731974725823, sessionid=0x101084ff21a0000, setting cluster-up flag (Was=false)
2024-11-19T00:05:27,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:05:27,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:05:27,340 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-19T00:05:27,342 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,45857,1731974725823
2024-11-19T00:05:27,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:05:27,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:05:27,353 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-19T00:05:27,355 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,45857,1731974725823
2024-11-19T00:05:27,362 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-19T00:05:27,424 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.HRegionServer(746): ClusterId : dc3b74e2-0fe7-4757-ba99-c0b2b65aae11
2024-11-19T00:05:27,428 DEBUG [RS:0;9c732c6543ff:40793 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-19T00:05:27,433 DEBUG [RS:0;9c732c6543ff:40793 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-19T00:05:27,433 DEBUG [RS:0;9c732c6543ff:40793 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-19T00:05:27,436 DEBUG [RS:0;9c732c6543ff:40793 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-19T00:05:27,437 DEBUG [RS:0;9c732c6543ff:40793 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35402214, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0
2024-11-19T00:05:27,447 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-19T00:05:27,452 DEBUG [RS:0;9c732c6543ff:40793 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9c732c6543ff:40793
2024-11-19T00:05:27,454 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-19T00:05:27,455 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-19T00:05:27,455 DEBUG [RS:0;9c732c6543ff:40793 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-19T00:05:27,457 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-19T00:05:27,457 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.HRegionServer(2659): reportForDuty to master=9c732c6543ff,45857,1731974725823 with port=40793, startcode=1731974726526
2024-11-19T00:05:27,464 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-19T00:05:27,471 DEBUG [RS:0;9c732c6543ff:40793 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-19T00:05:27,470 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9c732c6543ff,45857,1731974725823
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-19T00:05:27,478 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:05:27,478 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:05:27,478 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:05:27,478 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:05:27,479 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9c732c6543ff:0, corePoolSize=10, maxPoolSize=10
2024-11-19T00:05:27,479 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:05:27,479 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2
2024-11-19T00:05:27,479 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:05:27,480 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731974757480
2024-11-19T00:05:27,482 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-19T00:05:27,483 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-19T00:05:27,484 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:05:27,484 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-19T00:05:27,486 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-19T00:05:27,487 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-19T00:05:27,487 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-19T00:05:27,487 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-19T00:05:27,488 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:27,490 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:05:27,490 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-19T00:05:27,492 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-19T00:05:27,493 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-19T00:05:27,494 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-19T00:05:27,497 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-19T00:05:27,497 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-19T00:05:27,500 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974727498,5,FailOnTimeoutGroup]
2024-11-19T00:05:27,501 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974727500,5,FailOnTimeoutGroup]
2024-11-19T00:05:27,501 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:27,502 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-19T00:05:27,503 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:27,503 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:27,504 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741831_1007 (size=1321)
2024-11-19T00:05:27,504 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741831_1007 (size=1321)
2024-11-19T00:05:27,506 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-19T00:05:27,507 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4
2024-11-19T00:05:27,526 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741832_1008 (size=32)
2024-11-19T00:05:27,526 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741832_1008 (size=32)
2024-11-19T00:05:27,528 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:05:27,533 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T00:05:27,537 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T00:05:27,537 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:05:27,538 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:05:27,538 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T00:05:27,541 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T00:05:27,541 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:05:27,542 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:05:27,543 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T00:05:27,546 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T00:05:27,546 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:05:27,547 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:05:27,547 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T00:05:27,550 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T00:05:27,550 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:05:27,551 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:05:27,552 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T00:05:27,553 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740
2024-11-19T00:05:27,554 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740
2024-11-19T00:05:27,556 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57999, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-19T00:05:27,557 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T00:05:27,557 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T00:05:27,559 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T00:05:27,562 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T00:05:27,564 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45857 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9c732c6543ff,40793,1731974726526
2024-11-19T00:05:27,566 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:05:27,567 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=872853, jitterRate=0.10989028215408325}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T00:05:27,567 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45857 {}] master.ServerManager(517): Registering regionserver=9c732c6543ff,40793,1731974726526
2024-11-19T00:05:27,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731974727528Initializing all the Stores at 1731974727530 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974727531 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974727531Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974727532 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974727532Cleaning up temporary data from old regions at 1731974727557 (+25 ms)Region opened successfully at 1731974727571 (+14 ms)
2024-11-19T00:05:27,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T00:05:27,571 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T00:05:27,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T00:05:27,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T00:05:27,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T00:05:27,573 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T00:05:27,573 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731974727571Disabling compacts and flushes for region at 1731974727571Disabling writes for close at 1731974727572 (+1 ms)Writing region close event to WAL at 1731974727573 (+1 ms)Closed at 1731974727573
2024-11-19T00:05:27,578 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:05:27,578 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-19T00:05:27,584 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-19T00:05:27,585 DEBUG [RS:0;9c732c6543ff:40793 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4
2024-11-19T00:05:27,585 DEBUG [RS:0;9c732c6543ff:40793 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34343
2024-11-19T00:05:27,585 DEBUG [RS:0;9c732c6543ff:40793 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-19T00:05:27,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T00:05:27,590 DEBUG [RS:0;9c732c6543ff:40793 {}] zookeeper.ZKUtil(111): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9c732c6543ff,40793,1731974726526
2024-11-19T00:05:27,590 WARN  [RS:0;9c732c6543ff:40793 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T00:05:27,591 INFO  [RS:0;9c732c6543ff:40793 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:05:27,591 DEBUG [RS:0;9c732c6543ff:40793 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526
2024-11-19T00:05:27,593 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9c732c6543ff,40793,1731974726526]
2024-11-19T00:05:27,593 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T00:05:27,596 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-19T00:05:27,619 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-19T00:05:27,633 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-19T00:05:27,638 INFO  [RS:0;9c732c6543ff:40793 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T00:05:27,638 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:27,639 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-19T00:05:27,646 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-19T00:05:27,647 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:27,648 DEBUG [RS:0;9c732c6543ff:40793 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:05:27,648 DEBUG [RS:0;9c732c6543ff:40793 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:05:27,648 DEBUG [RS:0;9c732c6543ff:40793 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:05:27,648 DEBUG [RS:0;9c732c6543ff:40793 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:05:27,649 DEBUG [RS:0;9c732c6543ff:40793 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:05:27,649 DEBUG [RS:0;9c732c6543ff:40793 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2
2024-11-19T00:05:27,649 DEBUG [RS:0;9c732c6543ff:40793 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:05:27,649 DEBUG [RS:0;9c732c6543ff:40793 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:05:27,650 DEBUG [RS:0;9c732c6543ff:40793 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:05:27,650 DEBUG [RS:0;9c732c6543ff:40793 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:05:27,650 DEBUG [RS:0;9c732c6543ff:40793 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:05:27,650 DEBUG [RS:0;9c732c6543ff:40793 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:05:27,651 DEBUG [RS:0;9c732c6543ff:40793 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3
2024-11-19T00:05:27,651 DEBUG [RS:0;9c732c6543ff:40793 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3
2024-11-19T00:05:27,652 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:27,652 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:27,653 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:27,653 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:27,653 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:27,653 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,40793,1731974726526-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T00:05:27,672 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-19T00:05:27,674 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,40793,1731974726526-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:27,674 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:27,675 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.Replication(171): 9c732c6543ff,40793,1731974726526 started
2024-11-19T00:05:27,693 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:27,693 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.HRegionServer(1482): Serving as 9c732c6543ff,40793,1731974726526, RpcServer on 9c732c6543ff/172.17.0.2:40793, sessionid=0x101084ff21a0001
2024-11-19T00:05:27,694 DEBUG [RS:0;9c732c6543ff:40793 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-19T00:05:27,694 DEBUG [RS:0;9c732c6543ff:40793 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9c732c6543ff,40793,1731974726526
2024-11-19T00:05:27,694 DEBUG [RS:0;9c732c6543ff:40793 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,40793,1731974726526'
2024-11-19T00:05:27,694 DEBUG [RS:0;9c732c6543ff:40793 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-19T00:05:27,695 DEBUG [RS:0;9c732c6543ff:40793 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-19T00:05:27,696 DEBUG [RS:0;9c732c6543ff:40793 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-19T00:05:27,696 DEBUG [RS:0;9c732c6543ff:40793 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-19T00:05:27,697 DEBUG [RS:0;9c732c6543ff:40793 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9c732c6543ff,40793,1731974726526
2024-11-19T00:05:27,697 DEBUG [RS:0;9c732c6543ff:40793 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,40793,1731974726526'
2024-11-19T00:05:27,697 DEBUG [RS:0;9c732c6543ff:40793 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-19T00:05:27,697 DEBUG [RS:0;9c732c6543ff:40793 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-19T00:05:27,698 DEBUG [RS:0;9c732c6543ff:40793 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-19T00:05:27,698 INFO  [RS:0;9c732c6543ff:40793 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-19T00:05:27,698 INFO  [RS:0;9c732c6543ff:40793 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-19T00:05:27,747 WARN  [9c732c6543ff:45857 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-19T00:05:27,807 INFO  [RS:0;9c732c6543ff:40793 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C40793%2C1731974726526, suffix=, logDir=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526, archiveDir=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/oldWALs, maxLogs=32
2024-11-19T00:05:27,809 INFO  [RS:0;9c732c6543ff:40793 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C40793%2C1731974726526.1731974727809
2024-11-19T00:05:27,818 INFO  [RS:0;9c732c6543ff:40793 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974727809
2024-11-19T00:05:27,819 DEBUG [RS:0;9c732c6543ff:40793 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42447:42447),(127.0.0.1/127.0.0.1:37793:37793)]
2024-11-19T00:05:27,999 DEBUG [9c732c6543ff:45857 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-19T00:05:28,012 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9c732c6543ff,40793,1731974726526
2024-11-19T00:05:28,019 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,40793,1731974726526, state=OPENING
2024-11-19T00:05:28,024 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-19T00:05:28,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:05:28,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:05:28,027 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:05:28,027 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:05:28,028 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T00:05:28,030 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,40793,1731974726526}]
2024-11-19T00:05:28,206 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-19T00:05:28,211 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37039, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-19T00:05:28,222 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-19T00:05:28,223 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:05:28,227 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C40793%2C1731974726526.meta, suffix=.meta, logDir=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526, archiveDir=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/oldWALs, maxLogs=32
2024-11-19T00:05:28,230 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C40793%2C1731974726526.meta.1731974728229.meta
2024-11-19T00:05:28,238 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.meta.1731974728229.meta
2024-11-19T00:05:28,240 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37793:37793),(127.0.0.1/127.0.0.1:42447:42447)]
2024-11-19T00:05:28,241 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:05:28,243 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-19T00:05:28,246 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-19T00:05:28,252 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-19T00:05:28,256 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-19T00:05:28,257 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:05:28,257 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-19T00:05:28,257 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-19T00:05:28,260 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T00:05:28,262 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T00:05:28,262 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:05:28,263 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:05:28,263 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T00:05:28,265 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T00:05:28,265 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:05:28,265 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:05:28,266 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T00:05:28,267 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T00:05:28,267 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:05:28,268 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:05:28,268 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T00:05:28,269 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T00:05:28,269 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:05:28,270 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:05:28,270 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T00:05:28,271 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740
2024-11-19T00:05:28,274 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740
2024-11-19T00:05:28,276 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T00:05:28,276 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T00:05:28,277 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T00:05:28,279 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T00:05:28,281 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=704050, jitterRate=-0.10475465655326843}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T00:05:28,281 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-19T00:05:28,283 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731974728258Writing region info on filesystem at 1731974728258Initializing all the Stores at 1731974728260 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974728260Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974728260Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974728260Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974728260Cleaning up temporary data from old regions at 1731974728276 (+16 ms)Running coprocessor post-open hooks at 1731974728281 (+5 ms)Region opened successfully at 1731974728282 (+1 ms)
2024-11-19T00:05:28,290 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731974728197
2024-11-19T00:05:28,302 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-19T00:05:28,302 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-19T00:05:28,304 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,40793,1731974726526
2024-11-19T00:05:28,306 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,40793,1731974726526, state=OPEN
2024-11-19T00:05:28,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T00:05:28,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T00:05:28,311 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:05:28,311 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:05:28,312 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9c732c6543ff,40793,1731974726526
2024-11-19T00:05:28,318 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-19T00:05:28,319 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,40793,1731974726526 in 282 msec
2024-11-19T00:05:28,325 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-19T00:05:28,325 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 736 msec
2024-11-19T00:05:28,327 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:05:28,327 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-19T00:05:28,354 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T00:05:28,355 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,40793,1731974726526, seqNum=-1]
2024-11-19T00:05:28,380 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T00:05:28,382 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59439, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T00:05:28,403 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0070 sec
2024-11-19T00:05:28,403 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731974728403, completionTime=-1
2024-11-19T00:05:28,406 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-19T00:05:28,406 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-19T00:05:28,434 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-19T00:05:28,435 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731974788434
2024-11-19T00:05:28,435 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731974848435
2024-11-19T00:05:28,435 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec
2024-11-19T00:05:28,438 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,45857,1731974725823-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:28,438 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,45857,1731974725823-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:28,438 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,45857,1731974725823-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:28,440 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9c732c6543ff:45857, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:28,440 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:28,441 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:28,447 DEBUG [master/9c732c6543ff:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-19T00:05:28,471 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.794sec
2024-11-19T00:05:28,472 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-19T00:05:28,473 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-19T00:05:28,474 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-19T00:05:28,475 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-19T00:05:28,475 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-19T00:05:28,476 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,45857,1731974725823-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T00:05:28,476 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,45857,1731974725823-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-19T00:05:28,485 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-19T00:05:28,486 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-19T00:05:28,486 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,45857,1731974725823-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:05:28,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4731d90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:05:28,538 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-19T00:05:28,538 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-19T00:05:28,542 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9c732c6543ff,45857,-1 for getting cluster id
2024-11-19T00:05:28,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-19T00:05:28,554 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dc3b74e2-0fe7-4757-ba99-c0b2b65aae11'
2024-11-19T00:05:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-19T00:05:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dc3b74e2-0fe7-4757-ba99-c0b2b65aae11"

2024-11-19T00:05:28,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24713418, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:05:28,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9c732c6543ff,45857,-1]
2024-11-19T00:05:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-19T00:05:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:05:28,566 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-19T00:05:28,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eb3700d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:05:28,569 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T00:05:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,40793,1731974726526, seqNum=-1]
2024-11-19T00:05:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T00:05:28,579 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T00:05:28,601 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9c732c6543ff,45857,1731974725823
2024-11-19T00:05:28,601 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:05:28,610 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-19T00:05:28,614 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-19T00:05:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9c732c6543ff,45857,1731974725823
2024-11-19T00:05:28,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2e192e5e
2024-11-19T00:05:28,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-19T00:05:28,626 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-19T00:05:28,628 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-19T00:05:28,628 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-19T00:05:28,632 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T00:05:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-11-19T00:05:28,642 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-19T00:05:28,644 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-11-19T00:05:28,645 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:05:28,647 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-19T00:05:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T00:05:28,698 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741835_1011 (size=389)
2024-11-19T00:05:28,698 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741835_1011 (size=389)
2024-11-19T00:05:28,702 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 436ed0cb01438fffb85e2c3b3e8576b0, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4
2024-11-19T00:05:28,712 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741836_1012 (size=72)
2024-11-19T00:05:28,712 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741836_1012 (size=72)
2024-11-19T00:05:28,713 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:05:28,713 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 436ed0cb01438fffb85e2c3b3e8576b0, disabling compactions & flushes
2024-11-19T00:05:28,713 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.
2024-11-19T00:05:28,713 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.
2024-11-19T00:05:28,713 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0. after waiting 0 ms
2024-11-19T00:05:28,713 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.
2024-11-19T00:05:28,714 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.
2024-11-19T00:05:28,714 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 436ed0cb01438fffb85e2c3b3e8576b0:
Waiting for close lock at 1731974728713Disabling compacts and flushes for region at 1731974728713Disabling writes for close at 1731974728713Writing region close event to WAL at 1731974728713Closed at 1731974728713
2024-11-19T00:05:28,716 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-19T00:05:28,721 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731974728716"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731974728716"}]},"ts":"1731974728716"}
2024-11-19T00:05:28,727 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-19T00:05:28,729 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-19T00:05:28,732 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731974728729"}]},"ts":"1731974728729"}
2024-11-19T00:05:28,737 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-11-19T00:05:28,739 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=436ed0cb01438fffb85e2c3b3e8576b0, ASSIGN}]
2024-11-19T00:05:28,741 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=436ed0cb01438fffb85e2c3b3e8576b0, ASSIGN
2024-11-19T00:05:28,743 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=436ed0cb01438fffb85e2c3b3e8576b0, ASSIGN; state=OFFLINE, location=9c732c6543ff,40793,1731974726526; forceNewPlan=false, retain=false
2024-11-19T00:05:28,895 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=436ed0cb01438fffb85e2c3b3e8576b0, regionState=OPENING, regionLocation=9c732c6543ff,40793,1731974726526
2024-11-19T00:05:28,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=436ed0cb01438fffb85e2c3b3e8576b0, ASSIGN because future has completed
2024-11-19T00:05:28,901 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 436ed0cb01438fffb85e2c3b3e8576b0, server=9c732c6543ff,40793,1731974726526}]
2024-11-19T00:05:29,063 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.
2024-11-19T00:05:29,063 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 436ed0cb01438fffb85e2c3b3e8576b0, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:05:29,064 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 436ed0cb01438fffb85e2c3b3e8576b0
2024-11-19T00:05:29,064 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:05:29,064 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 436ed0cb01438fffb85e2c3b3e8576b0
2024-11-19T00:05:29,064 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 436ed0cb01438fffb85e2c3b3e8576b0
2024-11-19T00:05:29,067 INFO  [StoreOpener-436ed0cb01438fffb85e2c3b3e8576b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 436ed0cb01438fffb85e2c3b3e8576b0 
2024-11-19T00:05:29,069 INFO  [StoreOpener-436ed0cb01438fffb85e2c3b3e8576b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 436ed0cb01438fffb85e2c3b3e8576b0 columnFamilyName info
2024-11-19T00:05:29,069 DEBUG [StoreOpener-436ed0cb01438fffb85e2c3b3e8576b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:05:29,071 INFO  [StoreOpener-436ed0cb01438fffb85e2c3b3e8576b0-1 {}] regionserver.HStore(327): Store=436ed0cb01438fffb85e2c3b3e8576b0/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:05:29,071 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 436ed0cb01438fffb85e2c3b3e8576b0
2024-11-19T00:05:29,072 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0
2024-11-19T00:05:29,073 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0
2024-11-19T00:05:29,074 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 436ed0cb01438fffb85e2c3b3e8576b0
2024-11-19T00:05:29,074 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 436ed0cb01438fffb85e2c3b3e8576b0
2024-11-19T00:05:29,076 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 436ed0cb01438fffb85e2c3b3e8576b0
2024-11-19T00:05:29,080 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:05:29,081 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 436ed0cb01438fffb85e2c3b3e8576b0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=745438, jitterRate=-0.05212782323360443}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-19T00:05:29,081 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 436ed0cb01438fffb85e2c3b3e8576b0
2024-11-19T00:05:29,083 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 436ed0cb01438fffb85e2c3b3e8576b0:
Running coprocessor pre-open hook at 1731974729064Writing region info on filesystem at 1731974729064Initializing all the Stores at 1731974729066 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974729066Cleaning up temporary data from old regions at 1731974729074 (+8 ms)Running coprocessor post-open hooks at 1731974729081 (+7 ms)Region opened successfully at 1731974729082 (+1 ms)
2024-11-19T00:05:29,085 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0., pid=6, masterSystemTime=1731974729055
2024-11-19T00:05:29,089 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.
2024-11-19T00:05:29,089 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.
2024-11-19T00:05:29,090 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=436ed0cb01438fffb85e2c3b3e8576b0, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,40793,1731974726526
2024-11-19T00:05:29,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 436ed0cb01438fffb85e2c3b3e8576b0, server=9c732c6543ff,40793,1731974726526 because future has completed
2024-11-19T00:05:29,101 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-19T00:05:29,101 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 436ed0cb01438fffb85e2c3b3e8576b0, server=9c732c6543ff,40793,1731974726526 in 196 msec
2024-11-19T00:05:29,105 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-19T00:05:29,105 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=436ed0cb01438fffb85e2c3b3e8576b0, ASSIGN in 362 msec
2024-11-19T00:05:29,107 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-19T00:05:29,107 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731974729107"}]},"ts":"1731974729107"}
2024-11-19T00:05:29,111 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-11-19T00:05:29,130 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-19T00:05:29,134 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 496 msec
2024-11-19T00:05:33,760 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-19T00:05:33,806 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-19T00:05:33,807 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-11-19T00:05:36,253 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-19T00:05:36,254 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-19T00:05:36,256 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-19T00:05:36,256 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-19T00:05:36,258 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T00:05:36,258 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-19T00:05:36,258 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-19T00:05:36,258 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-19T00:05:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T00:05:38,724 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-11-19T00:05:38,727 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-11-19T00:05:38,734 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-11-19T00:05:38,734 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.
2024-11-19T00:05:38,735 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C40793%2C1731974726526.1731974738735
2024-11-19T00:05:38,745 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:05:38,745 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:05:38,745 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:05:38,745 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:05:38,746 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:05:38,746 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974727809 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974738735
2024-11-19T00:05:38,749 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42447:42447),(127.0.0.1/127.0.0.1:37793:37793)]
2024-11-19T00:05:38,750 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974727809 is not closed yet, will try archiving it next time
2024-11-19T00:05:38,751 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741833_1009 (size=451)
2024-11-19T00:05:38,752 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741833_1009 (size=451)
2024-11-19T00:05:38,753 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974727809 to hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/oldWALs/9c732c6543ff%2C40793%2C1731974726526.1731974727809
2024-11-19T00:05:38,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0., hostname=9c732c6543ff,40793,1731974726526, seqNum=2]
2024-11-19T00:05:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40793 {}] regionserver.HRegion(8855): Flush requested on 436ed0cb01438fffb85e2c3b3e8576b0
2024-11-19T00:05:50,799 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 436ed0cb01438fffb85e2c3b3e8576b0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T00:05:50,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp/info/c2f97258936d4f8e848c3de891d13639 is 1080, key is row0001/info:/1731974738762/Put/seqid=0
2024-11-19T00:05:50,875 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741838_1014 (size=12509)
2024-11-19T00:05:50,875 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741838_1014 (size=12509)
2024-11-19T00:05:50,876 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp/info/c2f97258936d4f8e848c3de891d13639
2024-11-19T00:05:50,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp/info/c2f97258936d4f8e848c3de891d13639 as hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/c2f97258936d4f8e848c3de891d13639
2024-11-19T00:05:50,933 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/c2f97258936d4f8e848c3de891d13639, entries=7, sequenceid=11, filesize=12.2 K
2024-11-19T00:05:50,943 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 436ed0cb01438fffb85e2c3b3e8576b0 in 141ms, sequenceid=11, compaction requested=false
2024-11-19T00:05:50,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 436ed0cb01438fffb85e2c3b3e8576b0:

2024-11-19T00:05:54,913 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-19T00:05:58,808 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C40793%2C1731974726526.1731974758807
2024-11-19T00:05:59,016 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK], DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK]]
2024-11-19T00:05:59,016 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:05:59,017 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:05:59,017 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:05:59,017 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:05:59,017 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:05:59,017 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974738735 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974758807
2024-11-19T00:05:59,018 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42447:42447),(127.0.0.1/127.0.0.1:37793:37793)]
2024-11-19T00:05:59,018 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974738735 is not closed yet, will try archiving it next time
2024-11-19T00:05:59,020 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741837_1013 (size=12399)
2024-11-19T00:05:59,020 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741837_1013 (size=12399)
2024-11-19T00:05:59,222 INFO  [FSHLog-0-hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4-prefix:9c732c6543ff,40793,1731974726526 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK], DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK]]
2024-11-19T00:06:01,426 INFO  [FSHLog-0-hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4-prefix:9c732c6543ff,40793,1731974726526 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK], DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK]]
2024-11-19T00:06:03,631 INFO  [FSHLog-0-hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4-prefix:9c732c6543ff,40793,1731974726526 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK], DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK]]
2024-11-19T00:06:05,835 INFO  [FSHLog-0-hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4-prefix:9c732c6543ff,40793,1731974726526 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK], DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK]]
2024-11-19T00:06:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40793 {}] regionserver.HRegion(8855): Flush requested on 436ed0cb01438fffb85e2c3b3e8576b0
2024-11-19T00:06:05,835 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 436ed0cb01438fffb85e2c3b3e8576b0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T00:06:06,037 INFO  [FSHLog-0-hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4-prefix:9c732c6543ff,40793,1731974726526 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK], DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK]]
2024-11-19T00:06:06,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp/info/503472a52b93485e92a70f0e3eb4fefe is 1080, key is row0008/info:/1731974752798/Put/seqid=0
2024-11-19T00:06:06,051 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741840_1016 (size=12509)
2024-11-19T00:06:06,051 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741840_1016 (size=12509)
2024-11-19T00:06:06,052 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp/info/503472a52b93485e92a70f0e3eb4fefe
2024-11-19T00:06:06,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp/info/503472a52b93485e92a70f0e3eb4fefe as hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/503472a52b93485e92a70f0e3eb4fefe
2024-11-19T00:06:06,071 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/503472a52b93485e92a70f0e3eb4fefe, entries=7, sequenceid=21, filesize=12.2 K
2024-11-19T00:06:06,273 INFO  [FSHLog-0-hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4-prefix:9c732c6543ff,40793,1731974726526 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK], DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK]]
2024-11-19T00:06:06,273 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 436ed0cb01438fffb85e2c3b3e8576b0 in 438ms, sequenceid=21, compaction requested=false
2024-11-19T00:06:06,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 436ed0cb01438fffb85e2c3b3e8576b0:

2024-11-19T00:06:06,274 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-11-19T00:06:06,274 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:06:06,275 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/c2f97258936d4f8e848c3de891d13639 because midkey is the same as first or last row
2024-11-19T00:06:08,039 INFO  [FSHLog-0-hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4-prefix:9c732c6543ff,40793,1731974726526 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK], DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK]]
2024-11-19T00:06:08,491 INFO  [master/9c732c6543ff:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-19T00:06:08,492 INFO  [master/9c732c6543ff:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-19T00:06:10,243 INFO  [FSHLog-0-hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4-prefix:9c732c6543ff,40793,1731974726526 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK], DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK]]
2024-11-19T00:06:10,245 WARN  [FSHLog-0-hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4-prefix:9c732c6543ff,40793,1731974726526 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK], DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK]]
2024-11-19T00:06:10,246 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9c732c6543ff%2C40793%2C1731974726526:(num 1731974758807) roll requested
2024-11-19T00:06:10,246 INFO  [regionserver/9c732c6543ff:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C40793%2C1731974726526.1731974770246
2024-11-19T00:06:10,454 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 205 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK], DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK]]
2024-11-19T00:06:10,454 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:10,454 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:10,454 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:10,455 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:10,455 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:10,455 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974758807 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974770246
2024-11-19T00:06:10,457 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741839_1015 (size=7739)
2024-11-19T00:06:10,458 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741839_1015 (size=7739)
2024-11-19T00:06:10,460 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37793:37793),(127.0.0.1/127.0.0.1:42447:42447)]
2024-11-19T00:06:10,461 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974758807 is not closed yet, will try archiving it next time
2024-11-19T00:06:10,461 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974738735 to hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/oldWALs/9c732c6543ff%2C40793%2C1731974726526.1731974738735
2024-11-19T00:06:12,447 INFO  [FSHLog-0-hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4-prefix:9c732c6543ff,40793,1731974726526 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK], DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK]]
2024-11-19T00:06:14,064 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 436ed0cb01438fffb85e2c3b3e8576b0, had cached 0 bytes from a total of 25018
2024-11-19T00:06:14,651 INFO  [FSHLog-0-hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4-prefix:9c732c6543ff,40793,1731974726526 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK], DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK]]
2024-11-19T00:06:16,855 INFO  [FSHLog-0-hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4-prefix:9c732c6543ff,40793,1731974726526 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK], DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK]]
2024-11-19T00:06:19,059 INFO  [FSHLog-0-hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4-prefix:9c732c6543ff,40793,1731974726526 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK], DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK]]
2024-11-19T00:06:21,061 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-19T00:06:21,061 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C40793%2C1731974726526.1731974781061
2024-11-19T00:06:24,914 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-19T00:06:26,070 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK], DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK]]
2024-11-19T00:06:26,073 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK], DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK]]
2024-11-19T00:06:26,073 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9c732c6543ff%2C40793%2C1731974726526:(num 1731974781061) roll requested
2024-11-19T00:06:26,073 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:26,073 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:26,073 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:26,074 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:26,074 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:26,074 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974770246 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974781061
2024-11-19T00:06:26,075 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37793:37793),(127.0.0.1/127.0.0.1:42447:42447)]
2024-11-19T00:06:26,075 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974770246 is not closed yet, will try archiving it next time
2024-11-19T00:06:26,075 INFO  [regionserver/9c732c6543ff:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C40793%2C1731974726526.1731974786075
2024-11-19T00:06:26,076 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741841_1017 (size=4753)
2024-11-19T00:06:26,077 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741841_1017 (size=4753)
2024-11-19T00:06:31,079 INFO  [FSHLog-0-hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4-prefix:9c732c6543ff,40793,1731974726526 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK], DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK]]
2024-11-19T00:06:31,079 WARN  [FSHLog-0-hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4-prefix:9c732c6543ff,40793,1731974726526 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK], DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK]]
2024-11-19T00:06:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40793 {}] regionserver.HRegion(8855): Flush requested on 436ed0cb01438fffb85e2c3b3e8576b0
2024-11-19T00:06:31,080 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 436ed0cb01438fffb85e2c3b3e8576b0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T00:06:31,087 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5008 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK], DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK]]
2024-11-19T00:06:31,087 WARN  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5008 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK], DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK]]
2024-11-19T00:06:33,081 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-19T00:06:36,082 INFO  [FSHLog-0-hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4-prefix:9c732c6543ff,40793,1731974726526 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK], DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK]]
2024-11-19T00:06:36,082 WARN  [FSHLog-0-hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4-prefix:9c732c6543ff,40793,1731974726526 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40913,DS-0229ef1f-3c2a-4337-8eb3-f1693e496632,DISK], DatanodeInfoWithStorage[127.0.0.1:43229,DS-2b9442c5-a77e-4909-b4c1-0c57e92fdb32,DISK]]
2024-11-19T00:06:36,082 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:36,082 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:36,083 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:36,083 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:36,083 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:36,083 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974781061 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974786075
2024-11-19T00:06:36,086 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741842_1018 (size=1569)
2024-11-19T00:06:36,086 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741842_1018 (size=1569)
2024-11-19T00:06:36,088 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42447:42447),(127.0.0.1/127.0.0.1:37793:37793)]
2024-11-19T00:06:36,088 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974781061 is not closed yet, will try archiving it next time
2024-11-19T00:06:36,088 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9c732c6543ff%2C40793%2C1731974726526:(num 1731974796088) roll requested
2024-11-19T00:06:36,089 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C40793%2C1731974726526.1731974796088
2024-11-19T00:06:36,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp/info/c2edb8f11b0749469e66dc38cce7a93d is 1080, key is row0015/info:/1731974767837/Put/seqid=0
2024-11-19T00:06:36,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741845_1021 (size=12509)
2024-11-19T00:06:36,098 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:36,098 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:36,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741845_1021 (size=12509)
2024-11-19T00:06:36,098 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:36,098 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:36,098 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:36,099 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974786075 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974796088
2024-11-19T00:06:36,099 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp/info/c2edb8f11b0749469e66dc38cce7a93d
2024-11-19T00:06:36,101 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741843_1019 (size=93)
2024-11-19T00:06:36,101 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741843_1019 (size=93)
2024-11-19T00:06:36,102 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974786075 to hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/oldWALs/9c732c6543ff%2C40793%2C1731974726526.1731974786075
2024-11-19T00:06:36,109 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37793:37793),(127.0.0.1/127.0.0.1:42447:42447)]
2024-11-19T00:06:36,110 INFO  [regionserver/9c732c6543ff:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C40793%2C1731974726526.1731974796110
2024-11-19T00:06:36,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp/info/c2edb8f11b0749469e66dc38cce7a93d as hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/c2edb8f11b0749469e66dc38cce7a93d
2024-11-19T00:06:36,130 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/c2edb8f11b0749469e66dc38cce7a93d, entries=7, sequenceid=31, filesize=12.2 K
2024-11-19T00:06:36,131 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:36,131 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:36,131 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:36,131 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:36,131 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:36,132 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974796088 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974796110
2024-11-19T00:06:36,133 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37793:37793),(127.0.0.1/127.0.0.1:42447:42447)]
2024-11-19T00:06:36,133 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974796088 is not closed yet, will try archiving it next time
2024-11-19T00:06:36,134 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=1.05 KB/1076 for 436ed0cb01438fffb85e2c3b3e8576b0 in 5055ms, sequenceid=31, compaction requested=true
2024-11-19T00:06:36,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 436ed0cb01438fffb85e2c3b3e8576b0:

2024-11-19T00:06:36,134 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-11-19T00:06:36,135 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:06:36,135 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/c2f97258936d4f8e848c3de891d13639 because midkey is the same as first or last row
2024-11-19T00:06:36,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 436ed0cb01438fffb85e2c3b3e8576b0:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T00:06:36,136 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741844_1020 (size=1258)
2024-11-19T00:06:36,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741844_1020 (size=1258)
2024-11-19T00:06:36,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:06:36,139 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T00:06:36,143 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T00:06:36,144 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.HStore(1541): 436ed0cb01438fffb85e2c3b3e8576b0/info is initiating minor compaction (all files)
2024-11-19T00:06:36,145 INFO  [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 436ed0cb01438fffb85e2c3b3e8576b0/info in TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.
2024-11-19T00:06:36,145 INFO  [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/c2f97258936d4f8e848c3de891d13639, hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/503472a52b93485e92a70f0e3eb4fefe, hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/c2edb8f11b0749469e66dc38cce7a93d] into tmpdir=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp, totalSize=36.6 K
2024-11-19T00:06:36,147 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2f97258936d4f8e848c3de891d13639, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731974738762
2024-11-19T00:06:36,148 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 503472a52b93485e92a70f0e3eb4fefe, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1731974752798
2024-11-19T00:06:36,148 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2edb8f11b0749469e66dc38cce7a93d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1731974767837
2024-11-19T00:06:36,195 INFO  [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 436ed0cb01438fffb85e2c3b3e8576b0#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T00:06:36,197 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp/info/407423409f774feb973ad27f130623c6 is 1080, key is row0001/info:/1731974738762/Put/seqid=0
2024-11-19T00:06:36,205 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741847_1023 (size=27710)
2024-11-19T00:06:36,206 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741847_1023 (size=27710)
2024-11-19T00:06:36,220 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp/info/407423409f774feb973ad27f130623c6 as hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/407423409f774feb973ad27f130623c6
2024-11-19T00:06:36,241 INFO  [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 436ed0cb01438fffb85e2c3b3e8576b0/info of 436ed0cb01438fffb85e2c3b3e8576b0 into 407423409f774feb973ad27f130623c6(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T00:06:36,241 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 436ed0cb01438fffb85e2c3b3e8576b0:

2024-11-19T00:06:36,244 INFO  [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0., storeName=436ed0cb01438fffb85e2c3b3e8576b0/info, priority=13, startTime=1731974796136; duration=0sec
2024-11-19T00:06:36,244 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-19T00:06:36,244 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:06:36,244 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/407423409f774feb973ad27f130623c6 because midkey is the same as first or last row
2024-11-19T00:06:36,245 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-19T00:06:36,245 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:06:36,245 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/407423409f774feb973ad27f130623c6 because midkey is the same as first or last row
2024-11-19T00:06:36,245 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-19T00:06:36,245 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:06:36,245 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/407423409f774feb973ad27f130623c6 because midkey is the same as first or last row
2024-11-19T00:06:36,245 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:06:36,245 DEBUG [RS:0;9c732c6543ff:40793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 436ed0cb01438fffb85e2c3b3e8576b0:info
2024-11-19T00:06:36,536 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974758807 to hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/oldWALs/9c732c6543ff%2C40793%2C1731974726526.1731974758807
2024-11-19T00:06:36,539 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974770246 to hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/oldWALs/9c732c6543ff%2C40793%2C1731974726526.1731974770246
2024-11-19T00:06:36,541 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/WALs/9c732c6543ff,40793,1731974726526/9c732c6543ff%2C40793%2C1731974726526.1731974781061 to hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/oldWALs/9c732c6543ff%2C40793%2C1731974726526.1731974781061
2024-11-19T00:06:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40793 {}] regionserver.HRegion(8855): Flush requested on 436ed0cb01438fffb85e2c3b3e8576b0
2024-11-19T00:06:48,137 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 436ed0cb01438fffb85e2c3b3e8576b0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T00:06:48,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp/info/e5eb6076e1424ba2aa959dc7b91b8671 is 1080, key is row0022/info:/1731974796111/Put/seqid=0
2024-11-19T00:06:48,151 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741848_1024 (size=12509)
2024-11-19T00:06:48,152 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741848_1024 (size=12509)
2024-11-19T00:06:48,152 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp/info/e5eb6076e1424ba2aa959dc7b91b8671
2024-11-19T00:06:48,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp/info/e5eb6076e1424ba2aa959dc7b91b8671 as hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/e5eb6076e1424ba2aa959dc7b91b8671
2024-11-19T00:06:48,173 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/e5eb6076e1424ba2aa959dc7b91b8671, entries=7, sequenceid=42, filesize=12.2 K
2024-11-19T00:06:48,175 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 436ed0cb01438fffb85e2c3b3e8576b0 in 37ms, sequenceid=42, compaction requested=false
2024-11-19T00:06:48,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 436ed0cb01438fffb85e2c3b3e8576b0:

2024-11-19T00:06:48,175 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-11-19T00:06:48,175 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:06:48,175 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/407423409f774feb973ad27f130623c6 because midkey is the same as first or last row
2024-11-19T00:06:54,914 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-19T00:06:56,149 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-19T00:06:56,150 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T00:06:56,150 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:06:56,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:06:56,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:06:56,156 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-19T00:06:56,156 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-19T00:06:56,156 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=272026373, stopped=false
2024-11-19T00:06:56,156 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9c732c6543ff,45857,1731974725823
2024-11-19T00:06:56,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T00:06:56,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T00:06:56,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:56,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:56,158 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T00:06:56,159 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T00:06:56,159 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:06:56,159 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:06:56,159 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:06:56,159 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:06:56,159 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9c732c6543ff,40793,1731974726526' *****
2024-11-19T00:06:56,159 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-19T00:06:56,160 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-19T00:06:56,160 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-19T00:06:56,160 INFO  [RS:0;9c732c6543ff:40793 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-19T00:06:56,160 INFO  [RS:0;9c732c6543ff:40793 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-19T00:06:56,161 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.HRegionServer(3091): Received CLOSE for 436ed0cb01438fffb85e2c3b3e8576b0
2024-11-19T00:06:56,161 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.HRegionServer(959): stopping server 9c732c6543ff,40793,1731974726526
2024-11-19T00:06:56,161 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T00:06:56,161 INFO  [RS:0;9c732c6543ff:40793 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9c732c6543ff:40793.
2024-11-19T00:06:56,161 DEBUG [RS:0;9c732c6543ff:40793 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:06:56,162 DEBUG [RS:0;9c732c6543ff:40793 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:06:56,162 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 436ed0cb01438fffb85e2c3b3e8576b0, disabling compactions & flushes
2024-11-19T00:06:56,162 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.
2024-11-19T00:06:56,162 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.
2024-11-19T00:06:56,162 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-19T00:06:56,162 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0. after waiting 0 ms
2024-11-19T00:06:56,162 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-19T00:06:56,162 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.
2024-11-19T00:06:56,162 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-19T00:06:56,162 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-19T00:06:56,162 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 436ed0cb01438fffb85e2c3b3e8576b0 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-11-19T00:06:56,162 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-19T00:06:56,163 DEBUG [RS:0;9c732c6543ff:40793 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 436ed0cb01438fffb85e2c3b3e8576b0=TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.}
2024-11-19T00:06:56,163 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T00:06:56,163 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T00:06:56,163 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T00:06:56,163 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T00:06:56,163 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T00:06:56,163 DEBUG [RS:0;9c732c6543ff:40793 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 436ed0cb01438fffb85e2c3b3e8576b0
2024-11-19T00:06:56,163 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-11-19T00:06:56,168 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp/info/1ee2edc7162c4521aaeac81350bb3216 is 1080, key is row0029/info:/1731974810140/Put/seqid=0
2024-11-19T00:06:56,178 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741849_1025 (size=8193)
2024-11-19T00:06:56,179 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741849_1025 (size=8193)
2024-11-19T00:06:56,179 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp/info/1ee2edc7162c4521aaeac81350bb3216
2024-11-19T00:06:56,189 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/.tmp/info/1ee2edc7162c4521aaeac81350bb3216 as hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/1ee2edc7162c4521aaeac81350bb3216
2024-11-19T00:06:56,192 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740/.tmp/info/8a31dc2a9cce4d35a59deb64d9d021d8 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0./info:regioninfo/1731974729090/Put/seqid=0
2024-11-19T00:06:56,197 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/1ee2edc7162c4521aaeac81350bb3216, entries=3, sequenceid=48, filesize=8.0 K
2024-11-19T00:06:56,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741850_1026 (size=7016)
2024-11-19T00:06:56,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741850_1026 (size=7016)
2024-11-19T00:06:56,198 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 436ed0cb01438fffb85e2c3b3e8576b0 in 36ms, sequenceid=48, compaction requested=true
2024-11-19T00:06:56,199 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740/.tmp/info/8a31dc2a9cce4d35a59deb64d9d021d8
2024-11-19T00:06:56,203 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/c2f97258936d4f8e848c3de891d13639, hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/503472a52b93485e92a70f0e3eb4fefe, hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/c2edb8f11b0749469e66dc38cce7a93d] to archive
2024-11-19T00:06:56,206 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-19T00:06:56,210 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/c2f97258936d4f8e848c3de891d13639 to hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/archive/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/c2f97258936d4f8e848c3de891d13639
2024-11-19T00:06:56,212 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/503472a52b93485e92a70f0e3eb4fefe to hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/archive/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/503472a52b93485e92a70f0e3eb4fefe
2024-11-19T00:06:56,214 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/c2edb8f11b0749469e66dc38cce7a93d to hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/archive/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/info/c2edb8f11b0749469e66dc38cce7a93d
2024-11-19T00:06:56,227 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740/.tmp/ns/408940dea421463facba848086b72425 is 43, key is default/ns:d/1731974728387/Put/seqid=0
2024-11-19T00:06:56,234 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741851_1027 (size=5153)
2024-11-19T00:06:56,235 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741851_1027 (size=5153)
2024-11-19T00:06:56,235 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740/.tmp/ns/408940dea421463facba848086b72425
2024-11-19T00:06:56,230 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=9c732c6543ff:45857 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-19T00:06:56,235 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [c2f97258936d4f8e848c3de891d13639=12509, 503472a52b93485e92a70f0e3eb4fefe=12509, c2edb8f11b0749469e66dc38cce7a93d=12509]
2024-11-19T00:06:56,243 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/default/TestLogRolling-testSlowSyncLogRolling/436ed0cb01438fffb85e2c3b3e8576b0/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-11-19T00:06:56,246 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.
2024-11-19T00:06:56,246 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 436ed0cb01438fffb85e2c3b3e8576b0:
Waiting for close lock at 1731974816161Running coprocessor pre-close hooks at 1731974816162 (+1 ms)Disabling compacts and flushes for region at 1731974816162Disabling writes for close at 1731974816162Obtaining lock to block concurrent updates at 1731974816162Preparing flush snapshotting stores in 436ed0cb01438fffb85e2c3b3e8576b0 at 1731974816162Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1731974816162Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0. at 1731974816164 (+2 ms)Flushing 436ed0cb01438fffb85e2c3b3e8576b0/info: creating writer at 1731974816164Flushing 436ed0cb01438fffb85e2c3b3e8576b0/info: appending metadata at 1731974816168 (+4 ms)Flushing 436ed0cb01438fffb85e2c3b3e8576b0/info: closing flushed file at 1731974816168Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4263c568: reopening flushed file at 1731974816188 (+20 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 436ed0cb01438fffb85e2c3b3e8576b0 in 36ms, sequenceid=48, compaction requested=true at 1731974816199 (+11 ms)Writing region close event to WAL at 1731974816237 (+38 ms)Running coprocessor post-close hooks at 1731974816244 (+7 ms)Closed at 1731974816246 (+2 ms)
2024-11-19T00:06:56,247 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1731974728627.436ed0cb01438fffb85e2c3b3e8576b0.
2024-11-19T00:06:56,253 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-19T00:06:56,260 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740/.tmp/table/ee716143f2244b6e816e336a453ea5f1 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1731974729107/Put/seqid=0
2024-11-19T00:06:56,266 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741852_1028 (size=5396)
2024-11-19T00:06:56,266 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741852_1028 (size=5396)
2024-11-19T00:06:56,267 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740/.tmp/table/ee716143f2244b6e816e336a453ea5f1
2024-11-19T00:06:56,276 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740/.tmp/info/8a31dc2a9cce4d35a59deb64d9d021d8 as hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740/info/8a31dc2a9cce4d35a59deb64d9d021d8
2024-11-19T00:06:56,283 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740/info/8a31dc2a9cce4d35a59deb64d9d021d8, entries=10, sequenceid=11, filesize=6.9 K
2024-11-19T00:06:56,285 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740/.tmp/ns/408940dea421463facba848086b72425 as hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740/ns/408940dea421463facba848086b72425
2024-11-19T00:06:56,292 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740/ns/408940dea421463facba848086b72425, entries=2, sequenceid=11, filesize=5.0 K
2024-11-19T00:06:56,293 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740/.tmp/table/ee716143f2244b6e816e336a453ea5f1 as hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740/table/ee716143f2244b6e816e336a453ea5f1
2024-11-19T00:06:56,301 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740/table/ee716143f2244b6e816e336a453ea5f1, entries=2, sequenceid=11, filesize=5.3 K
2024-11-19T00:06:56,303 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 140ms, sequenceid=11, compaction requested=false
2024-11-19T00:06:56,309 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-19T00:06:56,310 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T00:06:56,310 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T00:06:56,311 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731974816162Running coprocessor pre-close hooks at 1731974816162Disabling compacts and flushes for region at 1731974816163 (+1 ms)Disabling writes for close at 1731974816163Obtaining lock to block concurrent updates at 1731974816163Preparing flush snapshotting stores in 1588230740 at 1731974816163Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1731974816163Flushing stores of hbase:meta,,1.1588230740 at 1731974816164 (+1 ms)Flushing 1588230740/info: creating writer at 1731974816165 (+1 ms)Flushing 1588230740/info: appending metadata at 1731974816191 (+26 ms)Flushing 1588230740/info: closing flushed file at 1731974816191Flushing 1588230740/ns: creating writer at 1731974816206 (+15 ms)Flushing 1588230740/ns: appending metadata at 1731974816227 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731974816227Flushing 1588230740/table: creating writer at 1731974816244 (+17 ms)Flushing 1588230740/table: appending metadata at 1731974816259 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731974816259Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65e8778: reopening flushed file at 1731974816275 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ac75243: reopening flushed file at 1731974816284 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3097bcab: reopening flushed file at 1731974816293 (+9 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 140ms, sequenceid=11, compaction requested=false at 1731974816303 (+10 ms)Writing region close event to WAL at 1731974816305 (+2 ms)Running coprocessor post-close hooks at 1731974816310 (+5 ms)Closed at 1731974816310
2024-11-19T00:06:56,311 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-19T00:06:56,363 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.HRegionServer(976): stopping server 9c732c6543ff,40793,1731974726526; all regions closed.
2024-11-19T00:06:56,365 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:56,365 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:56,365 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:56,366 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:56,366 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:56,368 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741834_1010 (size=3066)
2024-11-19T00:06:56,369 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741834_1010 (size=3066)
2024-11-19T00:06:56,372 DEBUG [RS:0;9c732c6543ff:40793 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/oldWALs
2024-11-19T00:06:56,373 INFO  [RS:0;9c732c6543ff:40793 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9c732c6543ff%2C40793%2C1731974726526.meta:.meta(num 1731974728229)
2024-11-19T00:06:56,373 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:56,373 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:56,373 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:56,374 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:56,374 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:56,376 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741846_1022 (size=13040)
2024-11-19T00:06:56,377 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741846_1022 (size=13040)
2024-11-19T00:06:56,381 DEBUG [RS:0;9c732c6543ff:40793 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/oldWALs
2024-11-19T00:06:56,381 INFO  [RS:0;9c732c6543ff:40793 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9c732c6543ff%2C40793%2C1731974726526:(num 1731974796110)
2024-11-19T00:06:56,381 DEBUG [RS:0;9c732c6543ff:40793 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:06:56,381 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T00:06:56,381 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T00:06:56,381 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.ChoreService(370): Chore service for: regionserver/9c732c6543ff:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-19T00:06:56,381 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T00:06:56,381 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T00:06:56,382 INFO  [RS:0;9c732c6543ff:40793 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40793
2024-11-19T00:06:56,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T00:06:56,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9c732c6543ff,40793,1731974726526
2024-11-19T00:06:56,386 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T00:06:56,389 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9c732c6543ff,40793,1731974726526]
2024-11-19T00:06:56,390 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9c732c6543ff,40793,1731974726526 already deleted, retry=false
2024-11-19T00:06:56,390 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9c732c6543ff,40793,1731974726526 expired; onlineServers=0
2024-11-19T00:06:56,390 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9c732c6543ff,45857,1731974725823' *****
2024-11-19T00:06:56,391 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-19T00:06:56,391 INFO  [M:0;9c732c6543ff:45857 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T00:06:56,391 INFO  [M:0;9c732c6543ff:45857 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T00:06:56,391 DEBUG [M:0;9c732c6543ff:45857 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-19T00:06:56,391 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-19T00:06:56,391 DEBUG [M:0;9c732c6543ff:45857 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-19T00:06:56,391 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974727500 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974727500,5,FailOnTimeoutGroup]
2024-11-19T00:06:56,391 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974727498 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974727498,5,FailOnTimeoutGroup]
2024-11-19T00:06:56,391 INFO  [M:0;9c732c6543ff:45857 {}] hbase.ChoreService(370): Chore service for: master/9c732c6543ff:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-19T00:06:56,391 INFO  [M:0;9c732c6543ff:45857 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T00:06:56,392 DEBUG [M:0;9c732c6543ff:45857 {}] master.HMaster(1795): Stopping service threads
2024-11-19T00:06:56,392 INFO  [M:0;9c732c6543ff:45857 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-19T00:06:56,392 INFO  [M:0;9c732c6543ff:45857 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T00:06:56,392 INFO  [M:0;9c732c6543ff:45857 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-19T00:06:56,392 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-19T00:06:56,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-19T00:06:56,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:56,394 DEBUG [M:0;9c732c6543ff:45857 {}] zookeeper.ZKUtil(347): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-19T00:06:56,394 WARN  [M:0;9c732c6543ff:45857 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-19T00:06:56,395 INFO  [M:0;9c732c6543ff:45857 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/.lastflushedseqids
2024-11-19T00:06:56,408 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741853_1029 (size=130)
2024-11-19T00:06:56,408 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741853_1029 (size=130)
2024-11-19T00:06:56,409 INFO  [M:0;9c732c6543ff:45857 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-19T00:06:56,409 INFO  [M:0;9c732c6543ff:45857 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-19T00:06:56,409 DEBUG [M:0;9c732c6543ff:45857 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T00:06:56,409 INFO  [M:0;9c732c6543ff:45857 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:06:56,409 DEBUG [M:0;9c732c6543ff:45857 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:06:56,409 DEBUG [M:0;9c732c6543ff:45857 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T00:06:56,409 DEBUG [M:0;9c732c6543ff:45857 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:06:56,410 INFO  [M:0;9c732c6543ff:45857 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.04 KB heapSize=29.21 KB
2024-11-19T00:06:56,428 DEBUG [M:0;9c732c6543ff:45857 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/04b77b3b44104132b954f987487155a2 is 82, key is hbase:meta,,1/info:regioninfo/1731974728304/Put/seqid=0
2024-11-19T00:06:56,434 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741854_1030 (size=5672)
2024-11-19T00:06:56,435 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741854_1030 (size=5672)
2024-11-19T00:06:56,435 INFO  [M:0;9c732c6543ff:45857 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/04b77b3b44104132b954f987487155a2
2024-11-19T00:06:56,466 DEBUG [M:0;9c732c6543ff:45857 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f513ed6fd3a6490ab90bfa24cc63ed4b is 767, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731974729133/Put/seqid=0
2024-11-19T00:06:56,473 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741855_1031 (size=6248)
2024-11-19T00:06:56,473 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741855_1031 (size=6248)
2024-11-19T00:06:56,474 INFO  [M:0;9c732c6543ff:45857 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.43 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f513ed6fd3a6490ab90bfa24cc63ed4b
2024-11-19T00:06:56,481 INFO  [M:0;9c732c6543ff:45857 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f513ed6fd3a6490ab90bfa24cc63ed4b
2024-11-19T00:06:56,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:06:56,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40793-0x101084ff21a0001, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:06:56,489 INFO  [RS:0;9c732c6543ff:40793 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T00:06:56,489 INFO  [RS:0;9c732c6543ff:40793 {}] regionserver.HRegionServer(1031): Exiting; stopping=9c732c6543ff,40793,1731974726526; zookeeper connection closed.
2024-11-19T00:06:56,490 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@355626fa {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@355626fa
2024-11-19T00:06:56,490 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-19T00:06:56,498 DEBUG [M:0;9c732c6543ff:45857 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cbf03ac69f8e4d92970839288fece1d1 is 69, key is 9c732c6543ff,40793,1731974726526/rs:state/1731974727570/Put/seqid=0
2024-11-19T00:06:56,504 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741856_1032 (size=5156)
2024-11-19T00:06:56,504 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741856_1032 (size=5156)
2024-11-19T00:06:56,505 INFO  [M:0;9c732c6543ff:45857 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cbf03ac69f8e4d92970839288fece1d1
2024-11-19T00:06:56,528 DEBUG [M:0;9c732c6543ff:45857 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1e1e78bf40ed4f95830a79014f555a10 is 52, key is load_balancer_on/state:d/1731974728606/Put/seqid=0
2024-11-19T00:06:56,533 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741857_1033 (size=5056)
2024-11-19T00:06:56,534 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741857_1033 (size=5056)
2024-11-19T00:06:56,534 INFO  [M:0;9c732c6543ff:45857 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1e1e78bf40ed4f95830a79014f555a10
2024-11-19T00:06:56,542 DEBUG [M:0;9c732c6543ff:45857 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/04b77b3b44104132b954f987487155a2 as hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/04b77b3b44104132b954f987487155a2
2024-11-19T00:06:56,549 INFO  [M:0;9c732c6543ff:45857 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/04b77b3b44104132b954f987487155a2, entries=8, sequenceid=59, filesize=5.5 K
2024-11-19T00:06:56,550 DEBUG [M:0;9c732c6543ff:45857 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f513ed6fd3a6490ab90bfa24cc63ed4b as hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f513ed6fd3a6490ab90bfa24cc63ed4b
2024-11-19T00:06:56,556 INFO  [M:0;9c732c6543ff:45857 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f513ed6fd3a6490ab90bfa24cc63ed4b
2024-11-19T00:06:56,556 INFO  [M:0;9c732c6543ff:45857 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f513ed6fd3a6490ab90bfa24cc63ed4b, entries=6, sequenceid=59, filesize=6.1 K
2024-11-19T00:06:56,557 DEBUG [M:0;9c732c6543ff:45857 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cbf03ac69f8e4d92970839288fece1d1 as hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cbf03ac69f8e4d92970839288fece1d1
2024-11-19T00:06:56,563 INFO  [M:0;9c732c6543ff:45857 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cbf03ac69f8e4d92970839288fece1d1, entries=1, sequenceid=59, filesize=5.0 K
2024-11-19T00:06:56,564 DEBUG [M:0;9c732c6543ff:45857 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1e1e78bf40ed4f95830a79014f555a10 as hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1e1e78bf40ed4f95830a79014f555a10
2024-11-19T00:06:56,570 INFO  [M:0;9c732c6543ff:45857 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1e1e78bf40ed4f95830a79014f555a10, entries=1, sequenceid=59, filesize=4.9 K
2024-11-19T00:06:56,571 INFO  [M:0;9c732c6543ff:45857 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 162ms, sequenceid=59, compaction requested=false
2024-11-19T00:06:56,573 INFO  [M:0;9c732c6543ff:45857 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:06:56,573 DEBUG [M:0;9c732c6543ff:45857 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731974816409Disabling compacts and flushes for region at 1731974816409Disabling writes for close at 1731974816409Obtaining lock to block concurrent updates at 1731974816410 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731974816410Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23588, getHeapSize=29848, getOffHeapSize=0, getCellsCount=70 at 1731974816410Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731974816411 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731974816411Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731974816428 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731974816428Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731974816443 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731974816465 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731974816465Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731974816482 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731974816497 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731974816497Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731974816512 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731974816527 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731974816527Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bb08ecc: reopening flushed file at 1731974816541 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54e43919: reopening flushed file at 1731974816549 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75948a04: reopening flushed file at 1731974816556 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bab9902: reopening flushed file at 1731974816563 (+7 ms)Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 162ms, sequenceid=59, compaction requested=false at 1731974816571 (+8 ms)Writing region close event to WAL at 1731974816573 (+2 ms)Closed at 1731974816573
2024-11-19T00:06:56,574 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:56,574 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:56,574 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:56,574 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:56,574 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:56,577 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43229 is added to blk_1073741830_1006 (size=27985)
2024-11-19T00:06:56,577 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40913 is added to blk_1073741830_1006 (size=27985)
2024-11-19T00:06:56,578 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T00:06:56,578 INFO  [M:0;9c732c6543ff:45857 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-19T00:06:56,578 INFO  [M:0;9c732c6543ff:45857 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45857
2024-11-19T00:06:56,578 INFO  [M:0;9c732c6543ff:45857 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T00:06:56,680 INFO  [M:0;9c732c6543ff:45857 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T00:06:56,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:06:56,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x101084ff21a0000, quorum=127.0.0.1:54464, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:06:56,685 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:06:56,687 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:06:56,688 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:06:56,688 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:06:56,688 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/hadoop.log.dir/,STOPPED}
2024-11-19T00:06:56,691 WARN  [BP-1539334133-172.17.0.2-1731974722923 heartbeating to localhost/127.0.0.1:34343 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:06:56,691 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:06:56,691 WARN  [BP-1539334133-172.17.0.2-1731974722923 heartbeating to localhost/127.0.0.1:34343 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1539334133-172.17.0.2-1731974722923 (Datanode Uuid 30a8a02c-31c3-4278-8f24-cc4e0f6abb61) service to localhost/127.0.0.1:34343
2024-11-19T00:06:56,691 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:06:56,692 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/cluster_49ed2eb7-57e5-c679-c6b0-21c3fdb477d0/data/data3/current/BP-1539334133-172.17.0.2-1731974722923 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:06:56,692 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/cluster_49ed2eb7-57e5-c679-c6b0-21c3fdb477d0/data/data4/current/BP-1539334133-172.17.0.2-1731974722923 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:06:56,693 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:06:56,695 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:06:56,695 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:06:56,695 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:06:56,695 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:06:56,696 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/hadoop.log.dir/,STOPPED}
2024-11-19T00:06:56,697 WARN  [BP-1539334133-172.17.0.2-1731974722923 heartbeating to localhost/127.0.0.1:34343 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:06:56,697 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:06:56,697 WARN  [BP-1539334133-172.17.0.2-1731974722923 heartbeating to localhost/127.0.0.1:34343 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1539334133-172.17.0.2-1731974722923 (Datanode Uuid 3fb28dbf-0f80-4a88-86f4-b16ed1f50c31) service to localhost/127.0.0.1:34343
2024-11-19T00:06:56,697 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:06:56,698 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/cluster_49ed2eb7-57e5-c679-c6b0-21c3fdb477d0/data/data1/current/BP-1539334133-172.17.0.2-1731974722923 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:06:56,698 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/cluster_49ed2eb7-57e5-c679-c6b0-21c3fdb477d0/data/data2/current/BP-1539334133-172.17.0.2-1731974722923 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:06:56,699 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:06:56,708 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T00:06:56,709 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:06:56,709 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:06:56,709 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:06:56,709 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/hadoop.log.dir/,STOPPED}
2024-11-19T00:06:56,718 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-19T00:06:56,748 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-19T00:06:56,757 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=78 (was 12)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34343 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34343
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34343 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:34343
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: regionserver/9c732c6543ff:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34343
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/9c732c6543ff:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/9c732c6543ff:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: master/9c732c6543ff:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:34343
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34343
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34343 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@5f8f8226
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=402 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=99 (was 166), ProcessCount=11 (was 11), AvailableMemoryMB=4547 (was 5141)
2024-11-19T00:06:56,764 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=79, OpenFileDescriptor=402, MaxFileDescriptor=1048576, SystemLoadAverage=99, ProcessCount=11, AvailableMemoryMB=4547
2024-11-19T00:06:56,764 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-19T00:06:56,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/hadoop.log.dir so I do NOT create it in target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b
2024-11-19T00:06:56,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d882b870-0c01-63eb-e0c7-a25b4d0aedf9/hadoop.tmp.dir so I do NOT create it in target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b
2024-11-19T00:06:56,765 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/cluster_8ee6098e-9345-b71a-bbfc-63c4f8b0c959, deleteOnExit=true
2024-11-19T00:06:56,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-19T00:06:56,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/test.cache.data in system properties and HBase conf
2024-11-19T00:06:56,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/hadoop.tmp.dir in system properties and HBase conf
2024-11-19T00:06:56,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/hadoop.log.dir in system properties and HBase conf
2024-11-19T00:06:56,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-19T00:06:56,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-19T00:06:56,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-19T00:06:56,766 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-19T00:06:56,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-19T00:06:56,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-19T00:06:56,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-19T00:06:56,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T00:06:56,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-19T00:06:56,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-19T00:06:56,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T00:06:56,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T00:06:56,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-19T00:06:56,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/nfs.dump.dir in system properties and HBase conf
2024-11-19T00:06:56,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/java.io.tmpdir in system properties and HBase conf
2024-11-19T00:06:56,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T00:06:56,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-19T00:06:56,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-19T00:06:56,782 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T00:06:56,852 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:06:56,859 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:06:56,860 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:06:56,860 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:06:56,860 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T00:06:56,861 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:06:56,862 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1ae773f8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:06:56,862 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@630e1a46{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:06:56,987 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d195093{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/java.io.tmpdir/jetty-localhost-34501-hadoop-hdfs-3_4_1-tests_jar-_-any-3466402248478640379/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T00:06:56,988 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@476d81{HTTP/1.1, (http/1.1)}{localhost:34501}
2024-11-19T00:06:56,988 INFO  [Time-limited test {}] server.Server(415): Started @95996ms
2024-11-19T00:06:57,002 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T00:06:57,071 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:06:57,077 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:06:57,077 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:06:57,078 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:06:57,078 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T00:06:57,078 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7247ee1d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:06:57,078 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3270c9ae{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:06:57,195 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@15ba4d19{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/java.io.tmpdir/jetty-localhost-36299-hadoop-hdfs-3_4_1-tests_jar-_-any-4958906221046064507/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:06:57,195 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4d8874af{HTTP/1.1, (http/1.1)}{localhost:36299}
2024-11-19T00:06:57,195 INFO  [Time-limited test {}] server.Server(415): Started @96202ms
2024-11-19T00:06:57,197 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:06:57,231 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:06:57,235 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:06:57,235 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:06:57,236 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:06:57,236 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T00:06:57,236 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@64e5ce98{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:06:57,237 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3c30f553{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:06:57,287 WARN  [Thread-430 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/cluster_8ee6098e-9345-b71a-bbfc-63c4f8b0c959/data/data1/current/BP-1649493988-172.17.0.2-1731974816801/current, will proceed with Du for space computation calculation, 
2024-11-19T00:06:57,288 WARN  [Thread-431 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/cluster_8ee6098e-9345-b71a-bbfc-63c4f8b0c959/data/data2/current/BP-1649493988-172.17.0.2-1731974816801/current, will proceed with Du for space computation calculation, 
2024-11-19T00:06:57,306 WARN  [Thread-409 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:06:57,309 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf5a854b56a3137a9 with lease ID 0xf5f337123e6fe435: Processing first storage report for DS-38f2739c-b4b7-461a-9d5f-9cae062fc31a from datanode DatanodeRegistration(127.0.0.1:40361, datanodeUuid=61b591a3-8ed6-4c6a-8d47-ae0c7c06348a, infoPort=38829, infoSecurePort=0, ipcPort=34973, storageInfo=lv=-57;cid=testClusterID;nsid=1446901249;c=1731974816801)
2024-11-19T00:06:57,309 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf5a854b56a3137a9 with lease ID 0xf5f337123e6fe435: from storage DS-38f2739c-b4b7-461a-9d5f-9cae062fc31a node DatanodeRegistration(127.0.0.1:40361, datanodeUuid=61b591a3-8ed6-4c6a-8d47-ae0c7c06348a, infoPort=38829, infoSecurePort=0, ipcPort=34973, storageInfo=lv=-57;cid=testClusterID;nsid=1446901249;c=1731974816801), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:06:57,310 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf5a854b56a3137a9 with lease ID 0xf5f337123e6fe435: Processing first storage report for DS-a4c7060b-5608-46bd-a079-b34186bf98cc from datanode DatanodeRegistration(127.0.0.1:40361, datanodeUuid=61b591a3-8ed6-4c6a-8d47-ae0c7c06348a, infoPort=38829, infoSecurePort=0, ipcPort=34973, storageInfo=lv=-57;cid=testClusterID;nsid=1446901249;c=1731974816801)
2024-11-19T00:06:57,310 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf5a854b56a3137a9 with lease ID 0xf5f337123e6fe435: from storage DS-a4c7060b-5608-46bd-a079-b34186bf98cc node DatanodeRegistration(127.0.0.1:40361, datanodeUuid=61b591a3-8ed6-4c6a-8d47-ae0c7c06348a, infoPort=38829, infoSecurePort=0, ipcPort=34973, storageInfo=lv=-57;cid=testClusterID;nsid=1446901249;c=1731974816801), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-19T00:06:57,354 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2a489d78{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/java.io.tmpdir/jetty-localhost-39039-hadoop-hdfs-3_4_1-tests_jar-_-any-12383744933935474371/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:06:57,355 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3c67d530{HTTP/1.1, (http/1.1)}{localhost:39039}
2024-11-19T00:06:57,355 INFO  [Time-limited test {}] server.Server(415): Started @96362ms
2024-11-19T00:06:57,357 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:06:57,450 WARN  [Thread-456 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/cluster_8ee6098e-9345-b71a-bbfc-63c4f8b0c959/data/data3/current/BP-1649493988-172.17.0.2-1731974816801/current, will proceed with Du for space computation calculation, 
2024-11-19T00:06:57,451 WARN  [Thread-457 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/cluster_8ee6098e-9345-b71a-bbfc-63c4f8b0c959/data/data4/current/BP-1649493988-172.17.0.2-1731974816801/current, will proceed with Du for space computation calculation, 
2024-11-19T00:06:57,469 WARN  [Thread-445 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:06:57,471 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x44aad3731872de91 with lease ID 0xf5f337123e6fe436: Processing first storage report for DS-7ebe5629-cf81-4f54-abb4-34dcad26d4b6 from datanode DatanodeRegistration(127.0.0.1:43329, datanodeUuid=b8b4bbaa-15a9-4a77-a09c-e9df5197c289, infoPort=35497, infoSecurePort=0, ipcPort=37863, storageInfo=lv=-57;cid=testClusterID;nsid=1446901249;c=1731974816801)
2024-11-19T00:06:57,471 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x44aad3731872de91 with lease ID 0xf5f337123e6fe436: from storage DS-7ebe5629-cf81-4f54-abb4-34dcad26d4b6 node DatanodeRegistration(127.0.0.1:43329, datanodeUuid=b8b4bbaa-15a9-4a77-a09c-e9df5197c289, infoPort=35497, infoSecurePort=0, ipcPort=37863, storageInfo=lv=-57;cid=testClusterID;nsid=1446901249;c=1731974816801), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:06:57,471 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x44aad3731872de91 with lease ID 0xf5f337123e6fe436: Processing first storage report for DS-119911e5-2e9a-421d-b105-0b973d99392f from datanode DatanodeRegistration(127.0.0.1:43329, datanodeUuid=b8b4bbaa-15a9-4a77-a09c-e9df5197c289, infoPort=35497, infoSecurePort=0, ipcPort=37863, storageInfo=lv=-57;cid=testClusterID;nsid=1446901249;c=1731974816801)
2024-11-19T00:06:57,472 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x44aad3731872de91 with lease ID 0xf5f337123e6fe436: from storage DS-119911e5-2e9a-421d-b105-0b973d99392f node DatanodeRegistration(127.0.0.1:43329, datanodeUuid=b8b4bbaa-15a9-4a77-a09c-e9df5197c289, infoPort=35497, infoSecurePort=0, ipcPort=37863, storageInfo=lv=-57;cid=testClusterID;nsid=1446901249;c=1731974816801), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:06:57,490 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b
2024-11-19T00:06:57,493 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/cluster_8ee6098e-9345-b71a-bbfc-63c4f8b0c959/zookeeper_0, clientPort=57820, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/cluster_8ee6098e-9345-b71a-bbfc-63c4f8b0c959/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/cluster_8ee6098e-9345-b71a-bbfc-63c4f8b0c959/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-19T00:06:57,494 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57820
2024-11-19T00:06:57,494 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:06:57,496 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:06:57,506 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741825_1001 (size=7)
2024-11-19T00:06:57,507 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40361 is added to blk_1073741825_1001 (size=7)
2024-11-19T00:06:57,508 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6 with version=8
2024-11-19T00:06:57,508 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/hbase-staging
2024-11-19T00:06:57,511 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/9c732c6543ff:0 server-side Connection retries=45
2024-11-19T00:06:57,511 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:06:57,511 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T00:06:57,511 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T00:06:57,511 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:06:57,512 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T00:06:57,512 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-19T00:06:57,512 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T00:06:57,512 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37387
2024-11-19T00:06:57,514 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37387 connecting to ZooKeeper ensemble=127.0.0.1:57820
2024-11-19T00:06:57,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:373870x0, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T00:06:57,520 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37387-0x10108515b7f0000 connected
2024-11-19T00:06:57,536 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:06:57,538 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:06:57,540 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:06:57,541 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6, hbase.cluster.distributed=false
2024-11-19T00:06:57,542 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T00:06:57,543 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37387
2024-11-19T00:06:57,543 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37387
2024-11-19T00:06:57,543 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37387
2024-11-19T00:06:57,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37387
2024-11-19T00:06:57,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37387
2024-11-19T00:06:57,561 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/9c732c6543ff:0 server-side Connection retries=45
2024-11-19T00:06:57,561 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:06:57,561 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T00:06:57,561 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T00:06:57,561 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:06:57,561 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T00:06:57,561 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-19T00:06:57,561 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T00:06:57,562 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42405
2024-11-19T00:06:57,563 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42405 connecting to ZooKeeper ensemble=127.0.0.1:57820
2024-11-19T00:06:57,564 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:06:57,566 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:06:57,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:424050x0, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T00:06:57,572 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:424050x0, quorum=127.0.0.1:57820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:06:57,572 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42405-0x10108515b7f0001 connected
2024-11-19T00:06:57,572 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-19T00:06:57,573 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-19T00:06:57,573 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-19T00:06:57,575 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T00:06:57,575 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42405
2024-11-19T00:06:57,575 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42405
2024-11-19T00:06:57,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42405
2024-11-19T00:06:57,577 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42405
2024-11-19T00:06:57,577 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42405
2024-11-19T00:06:57,590 DEBUG [M:0;9c732c6543ff:37387 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9c732c6543ff:37387
2024-11-19T00:06:57,590 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9c732c6543ff,37387,1731974817511
2024-11-19T00:06:57,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:06:57,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:06:57,593 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9c732c6543ff,37387,1731974817511
2024-11-19T00:06:57,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:57,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-19T00:06:57,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:57,595 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-19T00:06:57,596 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9c732c6543ff,37387,1731974817511 from backup master directory
2024-11-19T00:06:57,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9c732c6543ff,37387,1731974817511
2024-11-19T00:06:57,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:06:57,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:06:57,597 WARN  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T00:06:57,597 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9c732c6543ff,37387,1731974817511
2024-11-19T00:06:57,602 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/hbase.id] with ID: 98c342fc-1b6f-4f21-9c99-16cd572ef120
2024-11-19T00:06:57,602 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/.tmp/hbase.id
2024-11-19T00:06:57,609 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741826_1002 (size=42)
2024-11-19T00:06:57,610 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40361 is added to blk_1073741826_1002 (size=42)
2024-11-19T00:06:57,610 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/.tmp/hbase.id]:[hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/hbase.id]
2024-11-19T00:06:57,627 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:06:57,627 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-19T00:06:57,630 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms.
2024-11-19T00:06:57,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:57,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:57,643 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40361 is added to blk_1073741827_1003 (size=196)
2024-11-19T00:06:57,643 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741827_1003 (size=196)
2024-11-19T00:06:57,646 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T00:06:57,647 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-19T00:06:57,648 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:06:57,658 INFO  [regionserver/9c732c6543ff:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T00:06:57,659 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40361 is added to blk_1073741828_1004 (size=1189)
2024-11-19T00:06:57,660 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741828_1004 (size=1189)
2024-11-19T00:06:57,661 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store
2024-11-19T00:06:57,668 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40361 is added to blk_1073741829_1005 (size=34)
2024-11-19T00:06:57,669 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741829_1005 (size=34)
2024-11-19T00:06:57,669 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:06:57,669 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T00:06:57,669 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:06:57,669 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:06:57,669 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T00:06:57,669 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:06:57,669 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:06:57,670 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731974817669Disabling compacts and flushes for region at 1731974817669Disabling writes for close at 1731974817669Writing region close event to WAL at 1731974817669Closed at 1731974817669
2024-11-19T00:06:57,671 WARN  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/.initializing
2024-11-19T00:06:57,671 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/WALs/9c732c6543ff,37387,1731974817511
2024-11-19T00:06:57,674 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C37387%2C1731974817511, suffix=, logDir=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/WALs/9c732c6543ff,37387,1731974817511, archiveDir=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/oldWALs, maxLogs=10
2024-11-19T00:06:57,674 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C37387%2C1731974817511.1731974817674
2024-11-19T00:06:57,680 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/WALs/9c732c6543ff,37387,1731974817511/9c732c6543ff%2C37387%2C1731974817511.1731974817674
2024-11-19T00:06:57,681 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38829:38829),(127.0.0.1/127.0.0.1:35497:35497)]
2024-11-19T00:06:57,685 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:06:57,685 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:06:57,685 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:06:57,685 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:06:57,688 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:06:57,690 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-19T00:06:57,690 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:06:57,690 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:06:57,691 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:06:57,692 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-19T00:06:57,692 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:06:57,693 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:06:57,693 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:06:57,695 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-19T00:06:57,696 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:06:57,696 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:06:57,696 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:06:57,698 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-19T00:06:57,698 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:06:57,698 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:06:57,698 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:06:57,699 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:06:57,700 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:06:57,701 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:06:57,701 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:06:57,702 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-19T00:06:57,703 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:06:57,706 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:06:57,706 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=708209, jitterRate=-0.09946602582931519}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-19T00:06:57,707 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731974817686Initializing all the Stores at 1731974817687 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974817687Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974817687Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974817687Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974817688 (+1 ms)Cleaning up temporary data from old regions at 1731974817701 (+13 ms)Region opened successfully at 1731974817707 (+6 ms)
2024-11-19T00:06:57,708 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-19T00:06:57,715 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c4407c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0
2024-11-19T00:06:57,716 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-19T00:06:57,716 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-19T00:06:57,716 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-19T00:06:57,716 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-19T00:06:57,717 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-19T00:06:57,717 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-19T00:06:57,717 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-19T00:06:57,720 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-19T00:06:57,721 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-19T00:06:57,723 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-19T00:06:57,723 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-19T00:06:57,724 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-19T00:06:57,725 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-19T00:06:57,726 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-19T00:06:57,727 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-19T00:06:57,728 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-19T00:06:57,729 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-19T00:06:57,731 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-19T00:06:57,733 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-19T00:06:57,734 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-19T00:06:57,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T00:06:57,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T00:06:57,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:57,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:57,737 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9c732c6543ff,37387,1731974817511, sessionid=0x10108515b7f0000, setting cluster-up flag (Was=false)
2024-11-19T00:06:57,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:57,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:57,747 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-19T00:06:57,748 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,37387,1731974817511
2024-11-19T00:06:57,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:57,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:57,757 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-19T00:06:57,758 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,37387,1731974817511
2024-11-19T00:06:57,759 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-19T00:06:57,761 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-19T00:06:57,762 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-19T00:06:57,762 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-19T00:06:57,762 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9c732c6543ff,37387,1731974817511
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-19T00:06:57,763 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:06:57,763 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:06:57,763 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:06:57,763 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:06:57,764 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9c732c6543ff:0, corePoolSize=10, maxPoolSize=10
2024-11-19T00:06:57,764 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:06:57,764 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2
2024-11-19T00:06:57,764 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:06:57,764 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731974847764
2024-11-19T00:06:57,765 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-19T00:06:57,765 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-19T00:06:57,765 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-19T00:06:57,765 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-19T00:06:57,765 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-19T00:06:57,765 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-19T00:06:57,765 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:57,765 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-19T00:06:57,766 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:06:57,766 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-19T00:06:57,766 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-19T00:06:57,766 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-19T00:06:57,766 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-19T00:06:57,766 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-19T00:06:57,766 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974817766,5,FailOnTimeoutGroup]
2024-11-19T00:06:57,767 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974817766,5,FailOnTimeoutGroup]
2024-11-19T00:06:57,767 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:57,767 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-19T00:06:57,767 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:57,767 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:57,767 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:06:57,767 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-19T00:06:57,774 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40361 is added to blk_1073741831_1007 (size=1321)
2024-11-19T00:06:57,774 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741831_1007 (size=1321)
2024-11-19T00:06:57,775 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-19T00:06:57,775 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6
2024-11-19T00:06:57,779 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.HRegionServer(746): ClusterId : 98c342fc-1b6f-4f21-9c99-16cd572ef120
2024-11-19T00:06:57,779 DEBUG [RS:0;9c732c6543ff:42405 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-19T00:06:57,781 DEBUG [RS:0;9c732c6543ff:42405 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-19T00:06:57,781 DEBUG [RS:0;9c732c6543ff:42405 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-19T00:06:57,782 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741832_1008 (size=32)
2024-11-19T00:06:57,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40361 is added to blk_1073741832_1008 (size=32)
2024-11-19T00:06:57,783 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:06:57,784 DEBUG [RS:0;9c732c6543ff:42405 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-19T00:06:57,784 DEBUG [RS:0;9c732c6543ff:42405 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276920ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0
2024-11-19T00:06:57,786 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T00:06:57,787 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T00:06:57,787 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:06:57,788 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:06:57,788 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T00:06:57,790 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T00:06:57,790 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:06:57,791 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:06:57,791 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T00:06:57,793 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T00:06:57,793 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:06:57,793 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:06:57,793 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T00:06:57,795 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T00:06:57,795 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:06:57,796 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:06:57,796 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T00:06:57,797 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/data/hbase/meta/1588230740
2024-11-19T00:06:57,797 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/data/hbase/meta/1588230740
2024-11-19T00:06:57,798 DEBUG [RS:0;9c732c6543ff:42405 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9c732c6543ff:42405
2024-11-19T00:06:57,798 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-19T00:06:57,798 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-19T00:06:57,798 DEBUG [RS:0;9c732c6543ff:42405 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-19T00:06:57,799 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.HRegionServer(2659): reportForDuty to master=9c732c6543ff,37387,1731974817511 with port=42405, startcode=1731974817560
2024-11-19T00:06:57,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T00:06:57,799 DEBUG [RS:0;9c732c6543ff:42405 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-19T00:06:57,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T00:06:57,800 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T00:06:57,801 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T00:06:57,803 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35623, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-11-19T00:06:57,804 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37387 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9c732c6543ff,42405,1731974817560
2024-11-19T00:06:57,804 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37387 {}] master.ServerManager(517): Registering regionserver=9c732c6543ff,42405,1731974817560
2024-11-19T00:06:57,804 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:06:57,805 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=781763, jitterRate=-0.005937859416007996}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T00:06:57,806 DEBUG [RS:0;9c732c6543ff:42405 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6
2024-11-19T00:06:57,806 DEBUG [RS:0;9c732c6543ff:42405 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43961
2024-11-19T00:06:57,806 DEBUG [RS:0;9c732c6543ff:42405 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-19T00:06:57,807 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731974817784Initializing all the Stores at 1731974817785 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974817785Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974817785Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974817785Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974817785Cleaning up temporary data from old regions at 1731974817799 (+14 ms)Region opened successfully at 1731974817807 (+8 ms)
2024-11-19T00:06:57,807 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T00:06:57,807 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T00:06:57,807 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T00:06:57,807 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T00:06:57,807 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T00:06:57,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T00:06:57,809 DEBUG [RS:0;9c732c6543ff:42405 {}] zookeeper.ZKUtil(111): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9c732c6543ff,42405,1731974817560
2024-11-19T00:06:57,809 WARN  [RS:0;9c732c6543ff:42405 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T00:06:57,809 INFO  [RS:0;9c732c6543ff:42405 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:06:57,809 DEBUG [RS:0;9c732c6543ff:42405 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/WALs/9c732c6543ff,42405,1731974817560
2024-11-19T00:06:57,809 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T00:06:57,810 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731974817807Disabling compacts and flushes for region at 1731974817807Disabling writes for close at 1731974817807Writing region close event to WAL at 1731974817809 (+2 ms)Closed at 1731974817809
2024-11-19T00:06:57,810 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9c732c6543ff,42405,1731974817560]
2024-11-19T00:06:57,811 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:06:57,811 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-19T00:06:57,811 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-19T00:06:57,813 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T00:06:57,815 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-19T00:06:57,815 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-19T00:06:57,819 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-19T00:06:57,819 INFO  [RS:0;9c732c6543ff:42405 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T00:06:57,819 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:57,820 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-19T00:06:57,821 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-19T00:06:57,821 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:57,821 DEBUG [RS:0;9c732c6543ff:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:06:57,821 DEBUG [RS:0;9c732c6543ff:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:06:57,821 DEBUG [RS:0;9c732c6543ff:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:06:57,821 DEBUG [RS:0;9c732c6543ff:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:06:57,821 DEBUG [RS:0;9c732c6543ff:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:06:57,821 DEBUG [RS:0;9c732c6543ff:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2
2024-11-19T00:06:57,821 DEBUG [RS:0;9c732c6543ff:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:06:57,821 DEBUG [RS:0;9c732c6543ff:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:06:57,822 DEBUG [RS:0;9c732c6543ff:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:06:57,822 DEBUG [RS:0;9c732c6543ff:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:06:57,822 DEBUG [RS:0;9c732c6543ff:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:06:57,822 DEBUG [RS:0;9c732c6543ff:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:06:57,822 DEBUG [RS:0;9c732c6543ff:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3
2024-11-19T00:06:57,822 DEBUG [RS:0;9c732c6543ff:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3
2024-11-19T00:06:57,824 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:57,824 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:57,824 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:57,824 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:57,824 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:57,824 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,42405,1731974817560-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T00:06:57,841 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-19T00:06:57,841 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,42405,1731974817560-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:57,841 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:57,841 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.Replication(171): 9c732c6543ff,42405,1731974817560 started
2024-11-19T00:06:57,857 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:57,857 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.HRegionServer(1482): Serving as 9c732c6543ff,42405,1731974817560, RpcServer on 9c732c6543ff/172.17.0.2:42405, sessionid=0x10108515b7f0001
2024-11-19T00:06:57,857 DEBUG [RS:0;9c732c6543ff:42405 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-19T00:06:57,857 DEBUG [RS:0;9c732c6543ff:42405 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9c732c6543ff,42405,1731974817560
2024-11-19T00:06:57,857 DEBUG [RS:0;9c732c6543ff:42405 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,42405,1731974817560'
2024-11-19T00:06:57,857 DEBUG [RS:0;9c732c6543ff:42405 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-19T00:06:57,858 DEBUG [RS:0;9c732c6543ff:42405 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-19T00:06:57,858 DEBUG [RS:0;9c732c6543ff:42405 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-19T00:06:57,858 DEBUG [RS:0;9c732c6543ff:42405 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-19T00:06:57,858 DEBUG [RS:0;9c732c6543ff:42405 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9c732c6543ff,42405,1731974817560
2024-11-19T00:06:57,858 DEBUG [RS:0;9c732c6543ff:42405 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,42405,1731974817560'
2024-11-19T00:06:57,858 DEBUG [RS:0;9c732c6543ff:42405 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-19T00:06:57,859 DEBUG [RS:0;9c732c6543ff:42405 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-19T00:06:57,859 DEBUG [RS:0;9c732c6543ff:42405 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-19T00:06:57,860 INFO  [RS:0;9c732c6543ff:42405 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-19T00:06:57,860 INFO  [RS:0;9c732c6543ff:42405 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-19T00:06:57,962 INFO  [RS:0;9c732c6543ff:42405 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C42405%2C1731974817560, suffix=, logDir=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/WALs/9c732c6543ff,42405,1731974817560, archiveDir=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/oldWALs, maxLogs=32
2024-11-19T00:06:57,964 INFO  [RS:0;9c732c6543ff:42405 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C42405%2C1731974817560.1731974817964
2024-11-19T00:06:57,966 WARN  [9c732c6543ff:37387 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-19T00:06:57,971 INFO  [RS:0;9c732c6543ff:42405 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/WALs/9c732c6543ff,42405,1731974817560/9c732c6543ff%2C42405%2C1731974817560.1731974817964
2024-11-19T00:06:57,973 DEBUG [RS:0;9c732c6543ff:42405 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38829:38829),(127.0.0.1/127.0.0.1:35497:35497)]
2024-11-19T00:06:58,216 DEBUG [9c732c6543ff:37387 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-19T00:06:58,217 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9c732c6543ff,42405,1731974817560
2024-11-19T00:06:58,218 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,42405,1731974817560, state=OPENING
2024-11-19T00:06:58,220 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-19T00:06:58,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:58,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:58,222 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T00:06:58,223 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:06:58,223 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:06:58,223 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,42405,1731974817560}]
2024-11-19T00:06:58,377 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-19T00:06:58,379 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59307, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-19T00:06:58,384 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-19T00:06:58,384 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:06:58,387 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C42405%2C1731974817560.meta, suffix=.meta, logDir=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/WALs/9c732c6543ff,42405,1731974817560, archiveDir=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/oldWALs, maxLogs=32
2024-11-19T00:06:58,390 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C42405%2C1731974817560.meta.1731974818390.meta
2024-11-19T00:06:58,396 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/WALs/9c732c6543ff,42405,1731974817560/9c732c6543ff%2C42405%2C1731974817560.meta.1731974818390.meta
2024-11-19T00:06:58,397 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38829:38829),(127.0.0.1/127.0.0.1:35497:35497)]
2024-11-19T00:06:58,398 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:06:58,398 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-19T00:06:58,398 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-19T00:06:58,398 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-19T00:06:58,398 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-19T00:06:58,399 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:06:58,399 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-19T00:06:58,399 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-19T00:06:58,400 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T00:06:58,402 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T00:06:58,402 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:06:58,402 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:06:58,402 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T00:06:58,403 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T00:06:58,403 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:06:58,404 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:06:58,404 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T00:06:58,405 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T00:06:58,405 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:06:58,405 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:06:58,406 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T00:06:58,407 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T00:06:58,407 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:06:58,407 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:06:58,407 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T00:06:58,408 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/data/hbase/meta/1588230740
2024-11-19T00:06:58,410 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/data/hbase/meta/1588230740
2024-11-19T00:06:58,411 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T00:06:58,411 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T00:06:58,412 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T00:06:58,413 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T00:06:58,415 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=691454, jitterRate=-0.1207713633775711}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T00:06:58,415 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-19T00:06:58,416 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731974818399Writing region info on filesystem at 1731974818399Initializing all the Stores at 1731974818400 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974818400Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974818400Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974818400Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974818400Cleaning up temporary data from old regions at 1731974818411 (+11 ms)Running coprocessor post-open hooks at 1731974818415 (+4 ms)Region opened successfully at 1731974818416 (+1 ms)
2024-11-19T00:06:58,417 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731974818376
2024-11-19T00:06:58,420 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-19T00:06:58,421 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-19T00:06:58,422 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,42405,1731974817560
2024-11-19T00:06:58,423 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,42405,1731974817560, state=OPEN
2024-11-19T00:06:58,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T00:06:58,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T00:06:58,428 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9c732c6543ff,42405,1731974817560
2024-11-19T00:06:58,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:06:58,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:06:58,432 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-19T00:06:58,432 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,42405,1731974817560 in 205 msec
2024-11-19T00:06:58,436 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-19T00:06:58,436 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 621 msec
2024-11-19T00:06:58,437 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:06:58,437 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-19T00:06:58,439 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T00:06:58,439 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,42405,1731974817560, seqNum=-1]
2024-11-19T00:06:58,439 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T00:06:58,441 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59943, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T00:06:58,448 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 686 msec
2024-11-19T00:06:58,448 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731974818448, completionTime=-1
2024-11-19T00:06:58,448 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-19T00:06:58,448 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-19T00:06:58,450 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-19T00:06:58,451 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731974878450
2024-11-19T00:06:58,451 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731974938451
2024-11-19T00:06:58,451 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-19T00:06:58,451 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,37387,1731974817511-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:58,451 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,37387,1731974817511-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:58,451 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,37387,1731974817511-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:58,451 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9c732c6543ff:37387, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:58,451 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:58,451 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:58,454 DEBUG [master/9c732c6543ff:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-19T00:06:58,457 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.860sec
2024-11-19T00:06:58,457 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-19T00:06:58,457 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-19T00:06:58,457 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-19T00:06:58,457 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-19T00:06:58,457 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-19T00:06:58,457 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,37387,1731974817511-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T00:06:58,457 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,37387,1731974817511-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-19T00:06:58,461 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-19T00:06:58,461 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-19T00:06:58,461 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,37387,1731974817511-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:06:58,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60dafc4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:06:58,480 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9c732c6543ff,37387,-1 for getting cluster id
2024-11-19T00:06:58,480 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-19T00:06:58,482 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98c342fc-1b6f-4f21-9c99-16cd572ef120'
2024-11-19T00:06:58,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-19T00:06:58,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98c342fc-1b6f-4f21-9c99-16cd572ef120"

2024-11-19T00:06:58,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@647d7b64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:06:58,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9c732c6543ff,37387,-1]
2024-11-19T00:06:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-19T00:06:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:06:58,486 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-19T00:06:58,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a89c49a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:06:58,488 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T00:06:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,42405,1731974817560, seqNum=-1]
2024-11-19T00:06:58,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T00:06:58,491 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T00:06:58,493 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9c732c6543ff,37387,1731974817511
2024-11-19T00:06:58,494 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:06:58,499 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-19T00:06:58,499 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-19T00:06:58,499 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T00:06:58,500 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:06:58,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:06:58,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:06:58,500 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-19T00:06:58,500 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-19T00:06:58,500 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=292436729, stopped=false
2024-11-19T00:06:58,500 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9c732c6543ff,37387,1731974817511
2024-11-19T00:06:58,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T00:06:58,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T00:06:58,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:58,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:58,502 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T00:06:58,503 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T00:06:58,503 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:06:58,503 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:06:58,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:06:58,503 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:06:58,503 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9c732c6543ff,42405,1731974817560' *****
2024-11-19T00:06:58,504 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-19T00:06:58,504 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-19T00:06:58,504 INFO  [RS:0;9c732c6543ff:42405 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-19T00:06:58,504 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-19T00:06:58,504 INFO  [RS:0;9c732c6543ff:42405 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-19T00:06:58,504 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.HRegionServer(959): stopping server 9c732c6543ff,42405,1731974817560
2024-11-19T00:06:58,504 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T00:06:58,504 INFO  [RS:0;9c732c6543ff:42405 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9c732c6543ff:42405.
2024-11-19T00:06:58,504 DEBUG [RS:0;9c732c6543ff:42405 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:06:58,504 DEBUG [RS:0;9c732c6543ff:42405 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:06:58,505 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-19T00:06:58,505 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-19T00:06:58,505 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-19T00:06:58,505 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-19T00:06:58,505 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-19T00:06:58,505 DEBUG [RS:0;9c732c6543ff:42405 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-19T00:06:58,505 DEBUG [RS:0;9c732c6543ff:42405 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-19T00:06:58,505 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T00:06:58,505 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T00:06:58,505 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T00:06:58,505 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T00:06:58,506 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T00:06:58,506 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-19T00:06:58,526 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/data/hbase/meta/1588230740/.tmp/ns/1dc301b587f14d1c93686bd9060965c2 is 43, key is default/ns:d/1731974818442/Put/seqid=0
2024-11-19T00:06:58,533 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741835_1011 (size=5153)
2024-11-19T00:06:58,533 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40361 is added to blk_1073741835_1011 (size=5153)
2024-11-19T00:06:58,534 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/data/hbase/meta/1588230740/.tmp/ns/1dc301b587f14d1c93686bd9060965c2
2024-11-19T00:06:58,542 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/data/hbase/meta/1588230740/.tmp/ns/1dc301b587f14d1c93686bd9060965c2 as hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/data/hbase/meta/1588230740/ns/1dc301b587f14d1c93686bd9060965c2
2024-11-19T00:06:58,550 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/data/hbase/meta/1588230740/ns/1dc301b587f14d1c93686bd9060965c2, entries=2, sequenceid=6, filesize=5.0 K
2024-11-19T00:06:58,551 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 45ms, sequenceid=6, compaction requested=false
2024-11-19T00:06:58,551 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-19T00:06:58,557 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-19T00:06:58,558 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T00:06:58,558 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T00:06:58,558 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731974818505Running coprocessor pre-close hooks at 1731974818505Disabling compacts and flushes for region at 1731974818505Disabling writes for close at 1731974818505Obtaining lock to block concurrent updates at 1731974818506 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731974818506Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731974818506Flushing stores of hbase:meta,,1.1588230740 at 1731974818507 (+1 ms)Flushing 1588230740/ns: creating writer at 1731974818507Flushing 1588230740/ns: appending metadata at 1731974818526 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731974818526Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d680c5f: reopening flushed file at 1731974818541 (+15 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 45ms, sequenceid=6, compaction requested=false at 1731974818551 (+10 ms)Writing region close event to WAL at 1731974818553 (+2 ms)Running coprocessor post-close hooks at 1731974818558 (+5 ms)Closed at 1731974818558
2024-11-19T00:06:58,558 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-19T00:06:58,705 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.HRegionServer(976): stopping server 9c732c6543ff,42405,1731974817560; all regions closed.
2024-11-19T00:06:58,706 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:58,706 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:58,706 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:58,707 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:58,707 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:58,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741834_1010 (size=1152)
2024-11-19T00:06:58,710 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40361 is added to blk_1073741834_1010 (size=1152)
2024-11-19T00:06:58,713 DEBUG [RS:0;9c732c6543ff:42405 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/oldWALs
2024-11-19T00:06:58,713 INFO  [RS:0;9c732c6543ff:42405 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9c732c6543ff%2C42405%2C1731974817560.meta:.meta(num 1731974818390)
2024-11-19T00:06:58,713 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:58,714 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:58,714 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:58,714 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:58,714 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:58,721 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741833_1009 (size=93)
2024-11-19T00:06:58,722 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40361 is added to blk_1073741833_1009 (size=93)
2024-11-19T00:06:58,724 DEBUG [RS:0;9c732c6543ff:42405 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/oldWALs
2024-11-19T00:06:58,724 INFO  [RS:0;9c732c6543ff:42405 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9c732c6543ff%2C42405%2C1731974817560:(num 1731974817964)
2024-11-19T00:06:58,724 DEBUG [RS:0;9c732c6543ff:42405 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:06:58,724 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T00:06:58,724 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T00:06:58,724 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.ChoreService(370): Chore service for: regionserver/9c732c6543ff:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-19T00:06:58,724 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T00:06:58,725 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T00:06:58,725 INFO  [RS:0;9c732c6543ff:42405 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42405
2024-11-19T00:06:58,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9c732c6543ff,42405,1731974817560
2024-11-19T00:06:58,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T00:06:58,727 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T00:06:58,729 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9c732c6543ff,42405,1731974817560]
2024-11-19T00:06:58,731 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9c732c6543ff,42405,1731974817560 already deleted, retry=false
2024-11-19T00:06:58,731 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9c732c6543ff,42405,1731974817560 expired; onlineServers=0
2024-11-19T00:06:58,731 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9c732c6543ff,37387,1731974817511' *****
2024-11-19T00:06:58,731 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-19T00:06:58,731 INFO  [M:0;9c732c6543ff:37387 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T00:06:58,731 INFO  [M:0;9c732c6543ff:37387 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T00:06:58,731 DEBUG [M:0;9c732c6543ff:37387 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-19T00:06:58,731 DEBUG [M:0;9c732c6543ff:37387 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-19T00:06:58,731 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-19T00:06:58,731 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974817766 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974817766,5,FailOnTimeoutGroup]
2024-11-19T00:06:58,731 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974817766 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974817766,5,FailOnTimeoutGroup]
2024-11-19T00:06:58,731 INFO  [M:0;9c732c6543ff:37387 {}] hbase.ChoreService(370): Chore service for: master/9c732c6543ff:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-19T00:06:58,732 INFO  [M:0;9c732c6543ff:37387 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T00:06:58,732 DEBUG [M:0;9c732c6543ff:37387 {}] master.HMaster(1795): Stopping service threads
2024-11-19T00:06:58,732 INFO  [M:0;9c732c6543ff:37387 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-19T00:06:58,732 INFO  [M:0;9c732c6543ff:37387 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T00:06:58,732 INFO  [M:0;9c732c6543ff:37387 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-19T00:06:58,732 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-19T00:06:58,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-19T00:06:58,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:58,733 DEBUG [M:0;9c732c6543ff:37387 {}] zookeeper.ZKUtil(347): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-19T00:06:58,733 WARN  [M:0;9c732c6543ff:37387 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-19T00:06:58,734 INFO  [M:0;9c732c6543ff:37387 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/.lastflushedseqids
2024-11-19T00:06:58,742 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40361 is added to blk_1073741836_1012 (size=99)
2024-11-19T00:06:58,742 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741836_1012 (size=99)
2024-11-19T00:06:58,743 INFO  [M:0;9c732c6543ff:37387 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-19T00:06:58,743 INFO  [M:0;9c732c6543ff:37387 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-19T00:06:58,743 DEBUG [M:0;9c732c6543ff:37387 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T00:06:58,743 INFO  [M:0;9c732c6543ff:37387 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:06:58,743 DEBUG [M:0;9c732c6543ff:37387 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:06:58,743 DEBUG [M:0;9c732c6543ff:37387 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T00:06:58,743 DEBUG [M:0;9c732c6543ff:37387 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:06:58,744 INFO  [M:0;9c732c6543ff:37387 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-19T00:06:58,770 DEBUG [M:0;9c732c6543ff:37387 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1dd98d0abaf748298de5689aa620a819 is 82, key is hbase:meta,,1/info:regioninfo/1731974818422/Put/seqid=0
2024-11-19T00:06:58,778 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40361 is added to blk_1073741837_1013 (size=5672)
2024-11-19T00:06:58,778 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741837_1013 (size=5672)
2024-11-19T00:06:58,778 INFO  [M:0;9c732c6543ff:37387 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1dd98d0abaf748298de5689aa620a819
2024-11-19T00:06:58,802 DEBUG [M:0;9c732c6543ff:37387 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/005389bbcf364fe4bc60b0db72a0d429 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731974818447/Put/seqid=0
2024-11-19T00:06:58,809 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741838_1014 (size=5275)
2024-11-19T00:06:58,809 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40361 is added to blk_1073741838_1014 (size=5275)
2024-11-19T00:06:58,810 INFO  [M:0;9c732c6543ff:37387 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/005389bbcf364fe4bc60b0db72a0d429
2024-11-19T00:06:58,829 INFO  [RS:0;9c732c6543ff:42405 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T00:06:58,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:06:58,829 INFO  [RS:0;9c732c6543ff:42405 {}] regionserver.HRegionServer(1031): Exiting; stopping=9c732c6543ff,42405,1731974817560; zookeeper connection closed.
2024-11-19T00:06:58,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10108515b7f0001, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:06:58,830 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2a156d0f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2a156d0f
2024-11-19T00:06:58,830 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-19T00:06:58,833 DEBUG [M:0;9c732c6543ff:37387 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/71efb14ee50c41269f2dbf445718c45d is 69, key is 9c732c6543ff,42405,1731974817560/rs:state/1731974817804/Put/seqid=0
2024-11-19T00:06:58,839 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741839_1015 (size=5156)
2024-11-19T00:06:58,840 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40361 is added to blk_1073741839_1015 (size=5156)
2024-11-19T00:06:58,841 INFO  [M:0;9c732c6543ff:37387 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/71efb14ee50c41269f2dbf445718c45d
2024-11-19T00:06:58,863 DEBUG [M:0;9c732c6543ff:37387 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/25816b6599d1425eacb8d05041b8d531 is 52, key is load_balancer_on/state:d/1731974818497/Put/seqid=0
2024-11-19T00:06:58,869 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40361 is added to blk_1073741840_1016 (size=5056)
2024-11-19T00:06:58,869 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741840_1016 (size=5056)
2024-11-19T00:06:58,870 INFO  [M:0;9c732c6543ff:37387 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/25816b6599d1425eacb8d05041b8d531
2024-11-19T00:06:58,876 DEBUG [M:0;9c732c6543ff:37387 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1dd98d0abaf748298de5689aa620a819 as hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1dd98d0abaf748298de5689aa620a819
2024-11-19T00:06:58,882 INFO  [M:0;9c732c6543ff:37387 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1dd98d0abaf748298de5689aa620a819, entries=8, sequenceid=29, filesize=5.5 K
2024-11-19T00:06:58,883 DEBUG [M:0;9c732c6543ff:37387 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/005389bbcf364fe4bc60b0db72a0d429 as hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/005389bbcf364fe4bc60b0db72a0d429
2024-11-19T00:06:58,890 INFO  [M:0;9c732c6543ff:37387 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/005389bbcf364fe4bc60b0db72a0d429, entries=3, sequenceid=29, filesize=5.2 K
2024-11-19T00:06:58,891 DEBUG [M:0;9c732c6543ff:37387 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/71efb14ee50c41269f2dbf445718c45d as hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/71efb14ee50c41269f2dbf445718c45d
2024-11-19T00:06:58,897 INFO  [M:0;9c732c6543ff:37387 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/71efb14ee50c41269f2dbf445718c45d, entries=1, sequenceid=29, filesize=5.0 K
2024-11-19T00:06:58,898 DEBUG [M:0;9c732c6543ff:37387 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/25816b6599d1425eacb8d05041b8d531 as hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/25816b6599d1425eacb8d05041b8d531
2024-11-19T00:06:58,903 INFO  [M:0;9c732c6543ff:37387 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43961/user/jenkins/test-data/d312f24e-03aa-b6ee-05a0-dc6658b9ffc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/25816b6599d1425eacb8d05041b8d531, entries=1, sequenceid=29, filesize=4.9 K
2024-11-19T00:06:58,905 INFO  [M:0;9c732c6543ff:37387 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 162ms, sequenceid=29, compaction requested=false
2024-11-19T00:06:58,906 INFO  [M:0;9c732c6543ff:37387 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:06:58,907 DEBUG [M:0;9c732c6543ff:37387 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731974818743Disabling compacts and flushes for region at 1731974818743Disabling writes for close at 1731974818743Obtaining lock to block concurrent updates at 1731974818744 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731974818744Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731974818744Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731974818745 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731974818745Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731974818769 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731974818769Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731974818785 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731974818802 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731974818802Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731974818816 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731974818833 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731974818833Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731974818847 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731974818863 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731974818863Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75b05e4e: reopening flushed file at 1731974818875 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@328d2459: reopening flushed file at 1731974818883 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29d22a82: reopening flushed file at 1731974818890 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21029215: reopening flushed file at 1731974818897 (+7 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 162ms, sequenceid=29, compaction requested=false at 1731974818905 (+8 ms)Writing region close event to WAL at 1731974818906 (+1 ms)Closed at 1731974818906
2024-11-19T00:06:58,907 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:58,907 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:58,907 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:58,907 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:58,908 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:06:58,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741830_1006 (size=10311)
2024-11-19T00:06:58,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40361 is added to blk_1073741830_1006 (size=10311)
2024-11-19T00:06:58,913 INFO  [M:0;9c732c6543ff:37387 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-19T00:06:58,913 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T00:06:58,913 INFO  [M:0;9c732c6543ff:37387 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37387
2024-11-19T00:06:58,914 INFO  [M:0;9c732c6543ff:37387 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T00:06:59,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:06:59,016 INFO  [M:0;9c732c6543ff:37387 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T00:06:59,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37387-0x10108515b7f0000, quorum=127.0.0.1:57820, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:06:59,020 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2a489d78{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:06:59,020 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3c67d530{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:06:59,021 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:06:59,021 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3c30f553{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:06:59,021 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@64e5ce98{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/hadoop.log.dir/,STOPPED}
2024-11-19T00:06:59,022 WARN  [BP-1649493988-172.17.0.2-1731974816801 heartbeating to localhost/127.0.0.1:43961 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:06:59,022 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:06:59,022 WARN  [BP-1649493988-172.17.0.2-1731974816801 heartbeating to localhost/127.0.0.1:43961 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1649493988-172.17.0.2-1731974816801 (Datanode Uuid b8b4bbaa-15a9-4a77-a09c-e9df5197c289) service to localhost/127.0.0.1:43961
2024-11-19T00:06:59,023 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:06:59,023 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/cluster_8ee6098e-9345-b71a-bbfc-63c4f8b0c959/data/data3/current/BP-1649493988-172.17.0.2-1731974816801 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:06:59,023 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/cluster_8ee6098e-9345-b71a-bbfc-63c4f8b0c959/data/data4/current/BP-1649493988-172.17.0.2-1731974816801 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:06:59,024 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:06:59,026 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@15ba4d19{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:06:59,026 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4d8874af{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:06:59,026 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:06:59,027 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3270c9ae{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:06:59,027 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7247ee1d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/hadoop.log.dir/,STOPPED}
2024-11-19T00:06:59,028 WARN  [BP-1649493988-172.17.0.2-1731974816801 heartbeating to localhost/127.0.0.1:43961 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:06:59,028 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:06:59,028 WARN  [BP-1649493988-172.17.0.2-1731974816801 heartbeating to localhost/127.0.0.1:43961 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1649493988-172.17.0.2-1731974816801 (Datanode Uuid 61b591a3-8ed6-4c6a-8d47-ae0c7c06348a) service to localhost/127.0.0.1:43961
2024-11-19T00:06:59,028 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:06:59,029 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/cluster_8ee6098e-9345-b71a-bbfc-63c4f8b0c959/data/data1/current/BP-1649493988-172.17.0.2-1731974816801 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:06:59,029 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/cluster_8ee6098e-9345-b71a-bbfc-63c4f8b0c959/data/data2/current/BP-1649493988-172.17.0.2-1731974816801 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:06:59,029 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:06:59,035 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d195093{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T00:06:59,036 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@476d81{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:06:59,036 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:06:59,036 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@630e1a46{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:06:59,037 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1ae773f8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/hadoop.log.dir/,STOPPED}
2024-11-19T00:06:59,044 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-19T00:06:59,066 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-19T00:06:59,066 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-19T00:06:59,066 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/hadoop.log.dir so I do NOT create it in target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848
2024-11-19T00:06:59,066 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6695d00b-606e-cfc6-a8ff-b42c42b2184b/hadoop.tmp.dir so I do NOT create it in target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848
2024-11-19T00:06:59,067 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d, deleteOnExit=true
2024-11-19T00:06:59,067 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-19T00:06:59,067 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/test.cache.data in system properties and HBase conf
2024-11-19T00:06:59,067 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.tmp.dir in system properties and HBase conf
2024-11-19T00:06:59,067 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.log.dir in system properties and HBase conf
2024-11-19T00:06:59,067 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-19T00:06:59,067 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-19T00:06:59,067 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-19T00:06:59,067 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-19T00:06:59,068 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-19T00:06:59,068 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-19T00:06:59,068 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-19T00:06:59,068 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T00:06:59,068 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-19T00:06:59,068 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-19T00:06:59,069 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T00:06:59,069 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T00:06:59,069 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-19T00:06:59,069 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/nfs.dump.dir in system properties and HBase conf
2024-11-19T00:06:59,069 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/java.io.tmpdir in system properties and HBase conf
2024-11-19T00:06:59,069 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T00:06:59,069 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-19T00:06:59,070 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-19T00:06:59,084 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T00:06:59,155 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:06:59,160 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:06:59,163 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:06:59,163 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:06:59,164 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T00:06:59,164 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:06:59,165 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@35c95cb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:06:59,165 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5d8a9c69{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:06:59,282 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@56ff2226{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/java.io.tmpdir/jetty-localhost-42185-hadoop-hdfs-3_4_1-tests_jar-_-any-16696626244351254647/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T00:06:59,283 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4b5c1b0d{HTTP/1.1, (http/1.1)}{localhost:42185}
2024-11-19T00:06:59,283 INFO  [Time-limited test {}] server.Server(415): Started @98290ms
2024-11-19T00:06:59,297 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T00:06:59,365 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:06:59,368 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:06:59,369 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:06:59,369 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:06:59,369 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T00:06:59,370 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5917cb43{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:06:59,370 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@259c861e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:06:59,485 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3150e6db{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/java.io.tmpdir/jetty-localhost-46293-hadoop-hdfs-3_4_1-tests_jar-_-any-2708014584649793426/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:06:59,485 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@539e83f9{HTTP/1.1, (http/1.1)}{localhost:46293}
2024-11-19T00:06:59,486 INFO  [Time-limited test {}] server.Server(415): Started @98493ms
2024-11-19T00:06:59,487 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:06:59,521 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:06:59,525 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:06:59,526 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:06:59,526 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:06:59,526 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T00:06:59,527 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6471b09b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:06:59,527 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fe8f7b0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:06:59,574 WARN  [Thread-649 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data1/current/BP-1245155782-172.17.0.2-1731974819103/current, will proceed with Du for space computation calculation, 
2024-11-19T00:06:59,575 WARN  [Thread-650 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data2/current/BP-1245155782-172.17.0.2-1731974819103/current, will proceed with Du for space computation calculation, 
2024-11-19T00:06:59,593 WARN  [Thread-628 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:06:59,596 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfd388a9c661c98fc with lease ID 0x2cc8ef2c933a09cb: Processing first storage report for DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae from datanode DatanodeRegistration(127.0.0.1:42365, datanodeUuid=5986ea1c-b94a-4f7a-ba8d-0f5e20139946, infoPort=45363, infoSecurePort=0, ipcPort=41299, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103)
2024-11-19T00:06:59,597 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfd388a9c661c98fc with lease ID 0x2cc8ef2c933a09cb: from storage DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae node DatanodeRegistration(127.0.0.1:42365, datanodeUuid=5986ea1c-b94a-4f7a-ba8d-0f5e20139946, infoPort=45363, infoSecurePort=0, ipcPort=41299, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:06:59,597 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfd388a9c661c98fc with lease ID 0x2cc8ef2c933a09cb: Processing first storage report for DS-cc424898-a335-43aa-9c9d-05971d9a6101 from datanode DatanodeRegistration(127.0.0.1:42365, datanodeUuid=5986ea1c-b94a-4f7a-ba8d-0f5e20139946, infoPort=45363, infoSecurePort=0, ipcPort=41299, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103)
2024-11-19T00:06:59,597 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfd388a9c661c98fc with lease ID 0x2cc8ef2c933a09cb: from storage DS-cc424898-a335-43aa-9c9d-05971d9a6101 node DatanodeRegistration(127.0.0.1:42365, datanodeUuid=5986ea1c-b94a-4f7a-ba8d-0f5e20139946, infoPort=45363, infoSecurePort=0, ipcPort=41299, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:06:59,646 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@78ab676e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/java.io.tmpdir/jetty-localhost-39085-hadoop-hdfs-3_4_1-tests_jar-_-any-3918351599584358632/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:06:59,647 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4478d7de{HTTP/1.1, (http/1.1)}{localhost:39085}
2024-11-19T00:06:59,647 INFO  [Time-limited test {}] server.Server(415): Started @98654ms
2024-11-19T00:06:59,649 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:06:59,735 WARN  [Thread-675 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data3/current/BP-1245155782-172.17.0.2-1731974819103/current, will proceed with Du for space computation calculation, 
2024-11-19T00:06:59,735 WARN  [Thread-676 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data4/current/BP-1245155782-172.17.0.2-1731974819103/current, will proceed with Du for space computation calculation, 
2024-11-19T00:06:59,753 WARN  [Thread-664 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:06:59,756 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd1bc0bcab97a6188 with lease ID 0x2cc8ef2c933a09cc: Processing first storage report for DS-ffef1479-e174-482b-84f8-9ded4c4a4f76 from datanode DatanodeRegistration(127.0.0.1:40107, datanodeUuid=3f4a546e-1f94-4f62-a530-2c54092a62e8, infoPort=38697, infoSecurePort=0, ipcPort=42001, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103)
2024-11-19T00:06:59,756 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd1bc0bcab97a6188 with lease ID 0x2cc8ef2c933a09cc: from storage DS-ffef1479-e174-482b-84f8-9ded4c4a4f76 node DatanodeRegistration(127.0.0.1:40107, datanodeUuid=3f4a546e-1f94-4f62-a530-2c54092a62e8, infoPort=38697, infoSecurePort=0, ipcPort=42001, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:06:59,756 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd1bc0bcab97a6188 with lease ID 0x2cc8ef2c933a09cc: Processing first storage report for DS-361069a9-3a24-40e5-9240-1dd6e4cc18f2 from datanode DatanodeRegistration(127.0.0.1:40107, datanodeUuid=3f4a546e-1f94-4f62-a530-2c54092a62e8, infoPort=38697, infoSecurePort=0, ipcPort=42001, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103)
2024-11-19T00:06:59,756 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd1bc0bcab97a6188 with lease ID 0x2cc8ef2c933a09cc: from storage DS-361069a9-3a24-40e5-9240-1dd6e4cc18f2 node DatanodeRegistration(127.0.0.1:40107, datanodeUuid=3f4a546e-1f94-4f62-a530-2c54092a62e8, infoPort=38697, infoSecurePort=0, ipcPort=42001, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:06:59,778 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848
2024-11-19T00:06:59,780 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/zookeeper_0, clientPort=49660, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-19T00:06:59,781 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49660
2024-11-19T00:06:59,782 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:06:59,783 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:06:59,793 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42365 is added to blk_1073741825_1001 (size=7)
2024-11-19T00:06:59,794 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40107 is added to blk_1073741825_1001 (size=7)
2024-11-19T00:06:59,795 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35 with version=8
2024-11-19T00:06:59,795 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/hbase-staging
2024-11-19T00:06:59,797 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/9c732c6543ff:0 server-side Connection retries=45
2024-11-19T00:06:59,797 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:06:59,797 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T00:06:59,798 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T00:06:59,798 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:06:59,798 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T00:06:59,798 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-19T00:06:59,798 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T00:06:59,799 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46287
2024-11-19T00:06:59,800 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46287 connecting to ZooKeeper ensemble=127.0.0.1:49660
2024-11-19T00:06:59,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:462870x0, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T00:06:59,806 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46287-0x1010851646e0000 connected
2024-11-19T00:06:59,822 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:06:59,823 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:06:59,825 INFO  [regionserver/9c732c6543ff:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T00:06:59,826 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:06:59,826 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35, hbase.cluster.distributed=false
2024-11-19T00:06:59,828 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T00:06:59,828 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46287
2024-11-19T00:06:59,829 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46287
2024-11-19T00:06:59,829 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46287
2024-11-19T00:06:59,831 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46287
2024-11-19T00:06:59,831 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46287
2024-11-19T00:06:59,848 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/9c732c6543ff:0 server-side Connection retries=45
2024-11-19T00:06:59,848 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:06:59,848 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T00:06:59,848 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T00:06:59,848 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:06:59,848 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T00:06:59,849 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-19T00:06:59,849 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T00:06:59,849 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42145
2024-11-19T00:06:59,851 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42145 connecting to ZooKeeper ensemble=127.0.0.1:49660
2024-11-19T00:06:59,851 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:06:59,854 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:06:59,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:421450x0, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T00:06:59,859 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42145-0x1010851646e0001 connected
2024-11-19T00:06:59,859 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:06:59,859 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-19T00:06:59,860 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-19T00:06:59,860 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-19T00:06:59,862 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T00:06:59,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42145
2024-11-19T00:06:59,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42145
2024-11-19T00:06:59,866 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42145
2024-11-19T00:06:59,866 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42145
2024-11-19T00:06:59,866 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42145
2024-11-19T00:06:59,879 DEBUG [M:0;9c732c6543ff:46287 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9c732c6543ff:46287
2024-11-19T00:06:59,880 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9c732c6543ff,46287,1731974819797
2024-11-19T00:06:59,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:06:59,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:06:59,882 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9c732c6543ff,46287,1731974819797
2024-11-19T00:06:59,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-19T00:06:59,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:59,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:59,885 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-19T00:06:59,885 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9c732c6543ff,46287,1731974819797 from backup master directory
2024-11-19T00:06:59,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:06:59,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9c732c6543ff,46287,1731974819797
2024-11-19T00:06:59,887 WARN  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T00:06:59,887 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9c732c6543ff,46287,1731974819797
2024-11-19T00:06:59,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:06:59,891 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/hbase.id] with ID: 2b98bb10-043a-41a9-9768-722f19d4ff85
2024-11-19T00:06:59,891 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/.tmp/hbase.id
2024-11-19T00:06:59,899 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40107 is added to blk_1073741826_1002 (size=42)
2024-11-19T00:06:59,900 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42365 is added to blk_1073741826_1002 (size=42)
2024-11-19T00:06:59,900 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/.tmp/hbase.id]:[hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/hbase.id]
2024-11-19T00:06:59,912 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:06:59,913 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-19T00:06:59,914 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-19T00:06:59,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:59,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:59,923 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40107 is added to blk_1073741827_1003 (size=196)
2024-11-19T00:06:59,923 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42365 is added to blk_1073741827_1003 (size=196)
2024-11-19T00:06:59,924 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T00:06:59,925 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-19T00:06:59,925 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:06:59,933 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42365 is added to blk_1073741828_1004 (size=1189)
2024-11-19T00:06:59,933 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40107 is added to blk_1073741828_1004 (size=1189)
2024-11-19T00:06:59,934 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store
2024-11-19T00:06:59,940 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40107 is added to blk_1073741829_1005 (size=34)
2024-11-19T00:06:59,941 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42365 is added to blk_1073741829_1005 (size=34)
2024-11-19T00:06:59,941 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:06:59,942 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T00:06:59,942 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:06:59,942 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:06:59,942 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T00:06:59,942 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:06:59,942 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:06:59,942 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731974819942Disabling compacts and flushes for region at 1731974819942Disabling writes for close at 1731974819942Writing region close event to WAL at 1731974819942Closed at 1731974819942
2024-11-19T00:06:59,943 WARN  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/.initializing
2024-11-19T00:06:59,943 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/WALs/9c732c6543ff,46287,1731974819797
2024-11-19T00:06:59,946 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C46287%2C1731974819797, suffix=, logDir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/WALs/9c732c6543ff,46287,1731974819797, archiveDir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/oldWALs, maxLogs=10
2024-11-19T00:06:59,946 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C46287%2C1731974819797.1731974819946
2024-11-19T00:06:59,951 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/WALs/9c732c6543ff,46287,1731974819797/9c732c6543ff%2C46287%2C1731974819797.1731974819946
2024-11-19T00:06:59,952 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38697:38697),(127.0.0.1/127.0.0.1:45363:45363)]
2024-11-19T00:06:59,953 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:06:59,953 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:06:59,953 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:06:59,953 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:06:59,955 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:06:59,956 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-19T00:06:59,956 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:06:59,957 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:06:59,957 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:06:59,958 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-19T00:06:59,958 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:06:59,959 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:06:59,959 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:06:59,960 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-19T00:06:59,960 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:06:59,961 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:06:59,961 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:06:59,962 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-19T00:06:59,962 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:06:59,963 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:06:59,963 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:06:59,963 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:06:59,964 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:06:59,966 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:06:59,966 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:06:59,966 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-19T00:06:59,967 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:06:59,970 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:06:59,970 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=748921, jitterRate=-0.047698333859443665}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-19T00:06:59,971 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731974819953Initializing all the Stores at 1731974819954 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974819954Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974819954Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974819955 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974819955Cleaning up temporary data from old regions at 1731974819966 (+11 ms)Region opened successfully at 1731974819971 (+5 ms)
2024-11-19T00:06:59,971 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-19T00:06:59,976 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@445e5365, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0
2024-11-19T00:06:59,977 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-19T00:06:59,977 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-19T00:06:59,977 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-19T00:06:59,977 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-19T00:06:59,977 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-19T00:06:59,978 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-19T00:06:59,978 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-19T00:06:59,980 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-19T00:06:59,981 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-19T00:06:59,982 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-19T00:06:59,982 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-19T00:06:59,983 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-19T00:06:59,984 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-19T00:06:59,984 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-19T00:06:59,985 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-19T00:06:59,987 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-19T00:06:59,987 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-19T00:06:59,989 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-19T00:06:59,991 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-19T00:06:59,992 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-19T00:06:59,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T00:06:59,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:59,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T00:06:59,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:59,994 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9c732c6543ff,46287,1731974819797, sessionid=0x1010851646e0000, setting cluster-up flag (Was=false)
2024-11-19T00:06:59,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:06:59,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:00,002 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-19T00:07:00,003 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,46287,1731974819797
2024-11-19T00:07:00,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:00,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:00,011 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-19T00:07:00,012 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,46287,1731974819797
2024-11-19T00:07:00,013 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-19T00:07:00,016 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-19T00:07:00,016 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-19T00:07:00,016 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-19T00:07:00,016 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9c732c6543ff,46287,1731974819797
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-19T00:07:00,018 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:07:00,018 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:07:00,018 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:07:00,018 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:07:00,018 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9c732c6543ff:0, corePoolSize=10, maxPoolSize=10
2024-11-19T00:07:00,018 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,018 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2
2024-11-19T00:07:00,018 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,020 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731974850020
2024-11-19T00:07:00,020 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-19T00:07:00,020 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-19T00:07:00,020 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-19T00:07:00,020 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-19T00:07:00,020 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-19T00:07:00,020 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-19T00:07:00,020 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,021 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-19T00:07:00,021 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-19T00:07:00,021 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-19T00:07:00,021 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:07:00,021 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-19T00:07:00,021 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-19T00:07:00,021 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-19T00:07:00,021 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974820021,5,FailOnTimeoutGroup]
2024-11-19T00:07:00,022 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974820021,5,FailOnTimeoutGroup]
2024-11-19T00:07:00,022 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,022 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-19T00:07:00,022 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,022 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,022 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:00,022 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-19T00:07:00,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42365 is added to blk_1073741831_1007 (size=1321)
2024-11-19T00:07:00,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40107 is added to blk_1073741831_1007 (size=1321)
2024-11-19T00:07:00,031 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-19T00:07:00,031 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35
2024-11-19T00:07:00,038 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42365 is added to blk_1073741832_1008 (size=32)
2024-11-19T00:07:00,038 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40107 is added to blk_1073741832_1008 (size=32)
2024-11-19T00:07:00,039 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:07:00,041 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T00:07:00,042 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T00:07:00,042 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:00,043 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:07:00,043 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T00:07:00,044 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T00:07:00,044 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:00,045 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:07:00,045 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T00:07:00,046 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T00:07:00,046 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:00,047 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:07:00,047 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T00:07:00,048 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T00:07:00,048 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:00,049 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:07:00,049 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T00:07:00,050 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740
2024-11-19T00:07:00,050 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740
2024-11-19T00:07:00,052 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T00:07:00,052 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T00:07:00,052 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T00:07:00,054 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T00:07:00,056 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:07:00,057 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=707497, jitterRate=-0.1003718227148056}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T00:07:00,059 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731974820039Initializing all the Stores at 1731974820040 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974820040Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974820040Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974820040Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974820041 (+1 ms)Cleaning up temporary data from old regions at 1731974820052 (+11 ms)Region opened successfully at 1731974820058 (+6 ms)
2024-11-19T00:07:00,059 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T00:07:00,059 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T00:07:00,059 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T00:07:00,059 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T00:07:00,059 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T00:07:00,059 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T00:07:00,060 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731974820059Disabling compacts and flushes for region at 1731974820059Disabling writes for close at 1731974820059Writing region close event to WAL at 1731974820059Closed at 1731974820059
2024-11-19T00:07:00,061 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:07:00,061 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-19T00:07:00,062 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-19T00:07:00,063 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T00:07:00,065 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-19T00:07:00,068 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(746): ClusterId : 2b98bb10-043a-41a9-9768-722f19d4ff85
2024-11-19T00:07:00,068 DEBUG [RS:0;9c732c6543ff:42145 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-19T00:07:00,071 DEBUG [RS:0;9c732c6543ff:42145 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-19T00:07:00,071 DEBUG [RS:0;9c732c6543ff:42145 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-19T00:07:00,073 DEBUG [RS:0;9c732c6543ff:42145 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-19T00:07:00,074 DEBUG [RS:0;9c732c6543ff:42145 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9e33cce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0
2024-11-19T00:07:00,087 DEBUG [RS:0;9c732c6543ff:42145 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9c732c6543ff:42145
2024-11-19T00:07:00,087 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-19T00:07:00,087 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-19T00:07:00,087 DEBUG [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-19T00:07:00,088 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(2659): reportForDuty to master=9c732c6543ff,46287,1731974819797 with port=42145, startcode=1731974819848
2024-11-19T00:07:00,088 DEBUG [RS:0;9c732c6543ff:42145 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-19T00:07:00,090 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45523, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-11-19T00:07:00,091 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46287 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9c732c6543ff,42145,1731974819848
2024-11-19T00:07:00,091 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46287 {}] master.ServerManager(517): Registering regionserver=9c732c6543ff,42145,1731974819848
2024-11-19T00:07:00,093 DEBUG [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35
2024-11-19T00:07:00,093 DEBUG [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43835
2024-11-19T00:07:00,093 DEBUG [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-19T00:07:00,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T00:07:00,095 DEBUG [RS:0;9c732c6543ff:42145 {}] zookeeper.ZKUtil(111): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9c732c6543ff,42145,1731974819848
2024-11-19T00:07:00,095 WARN  [RS:0;9c732c6543ff:42145 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T00:07:00,095 INFO  [RS:0;9c732c6543ff:42145 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:07:00,096 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9c732c6543ff,42145,1731974819848]
2024-11-19T00:07:00,096 DEBUG [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848
2024-11-19T00:07:00,100 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-19T00:07:00,102 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-19T00:07:00,103 INFO  [RS:0;9c732c6543ff:42145 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T00:07:00,103 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,103 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-19T00:07:00,104 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-19T00:07:00,104 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,104 DEBUG [RS:0;9c732c6543ff:42145 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,104 DEBUG [RS:0;9c732c6543ff:42145 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,104 DEBUG [RS:0;9c732c6543ff:42145 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,104 DEBUG [RS:0;9c732c6543ff:42145 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,104 DEBUG [RS:0;9c732c6543ff:42145 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,104 DEBUG [RS:0;9c732c6543ff:42145 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2
2024-11-19T00:07:00,104 DEBUG [RS:0;9c732c6543ff:42145 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,104 DEBUG [RS:0;9c732c6543ff:42145 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,104 DEBUG [RS:0;9c732c6543ff:42145 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,104 DEBUG [RS:0;9c732c6543ff:42145 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,104 DEBUG [RS:0;9c732c6543ff:42145 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,105 DEBUG [RS:0;9c732c6543ff:42145 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,105 DEBUG [RS:0;9c732c6543ff:42145 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3
2024-11-19T00:07:00,105 DEBUG [RS:0;9c732c6543ff:42145 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3
2024-11-19T00:07:00,105 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,105 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,105 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,105 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,105 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,105 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,42145,1731974819848-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T00:07:00,122 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-19T00:07:00,122 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,42145,1731974819848-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,122 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,122 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.Replication(171): 9c732c6543ff,42145,1731974819848 started
2024-11-19T00:07:00,138 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,138 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(1482): Serving as 9c732c6543ff,42145,1731974819848, RpcServer on 9c732c6543ff/172.17.0.2:42145, sessionid=0x1010851646e0001
2024-11-19T00:07:00,139 DEBUG [RS:0;9c732c6543ff:42145 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-19T00:07:00,139 DEBUG [RS:0;9c732c6543ff:42145 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9c732c6543ff,42145,1731974819848
2024-11-19T00:07:00,139 DEBUG [RS:0;9c732c6543ff:42145 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,42145,1731974819848'
2024-11-19T00:07:00,139 DEBUG [RS:0;9c732c6543ff:42145 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-19T00:07:00,139 DEBUG [RS:0;9c732c6543ff:42145 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-19T00:07:00,140 DEBUG [RS:0;9c732c6543ff:42145 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-19T00:07:00,140 DEBUG [RS:0;9c732c6543ff:42145 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-19T00:07:00,140 DEBUG [RS:0;9c732c6543ff:42145 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9c732c6543ff,42145,1731974819848
2024-11-19T00:07:00,140 DEBUG [RS:0;9c732c6543ff:42145 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,42145,1731974819848'
2024-11-19T00:07:00,140 DEBUG [RS:0;9c732c6543ff:42145 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-19T00:07:00,140 DEBUG [RS:0;9c732c6543ff:42145 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-19T00:07:00,141 DEBUG [RS:0;9c732c6543ff:42145 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-19T00:07:00,141 INFO  [RS:0;9c732c6543ff:42145 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-19T00:07:00,141 INFO  [RS:0;9c732c6543ff:42145 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-19T00:07:00,215 WARN  [9c732c6543ff:46287 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-19T00:07:00,244 INFO  [RS:0;9c732c6543ff:42145 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C42145%2C1731974819848, suffix=, logDir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848, archiveDir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/oldWALs, maxLogs=32
2024-11-19T00:07:00,245 INFO  [RS:0;9c732c6543ff:42145 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C42145%2C1731974819848.1731974820245
2024-11-19T00:07:00,252 INFO  [RS:0;9c732c6543ff:42145 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
2024-11-19T00:07:00,253 DEBUG [RS:0;9c732c6543ff:42145 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45363:45363),(127.0.0.1/127.0.0.1:38697:38697)]
2024-11-19T00:07:00,465 DEBUG [9c732c6543ff:46287 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-19T00:07:00,466 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9c732c6543ff,42145,1731974819848
2024-11-19T00:07:00,467 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,42145,1731974819848, state=OPENING
2024-11-19T00:07:00,469 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-19T00:07:00,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:00,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:00,471 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T00:07:00,472 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,42145,1731974819848}]
2024-11-19T00:07:00,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:07:00,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:07:00,625 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-19T00:07:00,628 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35905, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-19T00:07:00,635 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-19T00:07:00,635 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:07:00,638 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C42145%2C1731974819848.meta, suffix=.meta, logDir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848, archiveDir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/oldWALs, maxLogs=32
2024-11-19T00:07:00,639 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
2024-11-19T00:07:00,651 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
2024-11-19T00:07:00,657 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45363:45363),(127.0.0.1/127.0.0.1:38697:38697)]
2024-11-19T00:07:00,662 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:07:00,662 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-19T00:07:00,662 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-19T00:07:00,662 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-19T00:07:00,663 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-19T00:07:00,663 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:07:00,663 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-19T00:07:00,663 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-19T00:07:00,666 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T00:07:00,667 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T00:07:00,667 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:00,668 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:07:00,668 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T00:07:00,674 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T00:07:00,675 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:00,675 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:07:00,675 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T00:07:00,676 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T00:07:00,677 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:00,677 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:07:00,677 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T00:07:00,680 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T00:07:00,680 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:00,680 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:07:00,681 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T00:07:00,682 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740
2024-11-19T00:07:00,683 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740
2024-11-19T00:07:00,685 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T00:07:00,685 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T00:07:00,686 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T00:07:00,689 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T00:07:00,690 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=845561, jitterRate=0.07518696784973145}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T00:07:00,690 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-19T00:07:00,692 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731974820663Writing region info on filesystem at 1731974820663Initializing all the Stores at 1731974820665 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974820665Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974820666 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974820666Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974820666Cleaning up temporary data from old regions at 1731974820685 (+19 ms)Running coprocessor post-open hooks at 1731974820690 (+5 ms)Region opened successfully at 1731974820692 (+2 ms)
2024-11-19T00:07:00,693 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731974820625
2024-11-19T00:07:00,697 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-19T00:07:00,697 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-19T00:07:00,698 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,42145,1731974819848
2024-11-19T00:07:00,700 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,42145,1731974819848, state=OPEN
2024-11-19T00:07:00,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T00:07:00,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T00:07:00,705 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9c732c6543ff,42145,1731974819848
2024-11-19T00:07:00,705 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:07:00,705 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:07:00,709 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-19T00:07:00,709 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,42145,1731974819848 in 234 msec
2024-11-19T00:07:00,713 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-19T00:07:00,713 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 647 msec
2024-11-19T00:07:00,715 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:07:00,715 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-19T00:07:00,717 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T00:07:00,717 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,42145,1731974819848, seqNum=-1]
2024-11-19T00:07:00,717 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T00:07:00,719 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45783, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T00:07:00,727 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 710 msec
2024-11-19T00:07:00,727 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731974820727, completionTime=-1
2024-11-19T00:07:00,727 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-19T00:07:00,727 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-19T00:07:00,729 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-19T00:07:00,729 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731974880729
2024-11-19T00:07:00,729 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731974940729
2024-11-19T00:07:00,729 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-19T00:07:00,730 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,46287,1731974819797-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,730 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,46287,1731974819797-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,730 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,46287,1731974819797-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,730 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9c732c6543ff:46287, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,730 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,731 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,733 DEBUG [master/9c732c6543ff:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-19T00:07:00,736 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.849sec
2024-11-19T00:07:00,736 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-19T00:07:00,736 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-19T00:07:00,736 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-19T00:07:00,736 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-19T00:07:00,736 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-19T00:07:00,736 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,46287,1731974819797-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T00:07:00,736 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,46287,1731974819797-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-19T00:07:00,740 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-19T00:07:00,740 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-19T00:07:00,740 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,46287,1731974819797-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3efce601, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:07:00,769 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9c732c6543ff,46287,-1 for getting cluster id
2024-11-19T00:07:00,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-19T00:07:00,771 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2b98bb10-043a-41a9-9768-722f19d4ff85'
2024-11-19T00:07:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-19T00:07:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2b98bb10-043a-41a9-9768-722f19d4ff85"

2024-11-19T00:07:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fa7e7c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:07:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9c732c6543ff,46287,-1]
2024-11-19T00:07:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-19T00:07:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:07:00,775 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-19T00:07:00,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c279b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:07:00,776 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T00:07:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,42145,1731974819848, seqNum=-1]
2024-11-19T00:07:00,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T00:07:00,779 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T00:07:00,781 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9c732c6543ff,46287,1731974819797
2024-11-19T00:07:00,781 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:07:00,786 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-19T00:07:00,803 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/9c732c6543ff:0 server-side Connection retries=45
2024-11-19T00:07:00,803 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:07:00,803 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T00:07:00,803 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T00:07:00,803 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:07:00,803 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T00:07:00,803 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-19T00:07:00,803 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T00:07:00,804 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42069
2024-11-19T00:07:00,805 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42069 connecting to ZooKeeper ensemble=127.0.0.1:49660
2024-11-19T00:07:00,806 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:07:00,808 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:07:00,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:420690x0, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T00:07:00,814 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:420690x0, quorum=127.0.0.1:49660, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-11-19T00:07:00,814 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-11-19T00:07:00,816 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-19T00:07:00,823 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42069-0x1010851646e0002 connected
2024-11-19T00:07:00,824 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-19T00:07:00,825 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:42069-0x1010851646e0002, quorum=127.0.0.1:49660, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-19T00:07:00,829 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42069-0x1010851646e0002, quorum=127.0.0.1:49660, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T00:07:00,831 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42069
2024-11-19T00:07:00,831 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42069
2024-11-19T00:07:00,832 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42069
2024-11-19T00:07:00,832 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42069
2024-11-19T00:07:00,833 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42069
2024-11-19T00:07:00,838 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.HRegionServer(746): ClusterId : 2b98bb10-043a-41a9-9768-722f19d4ff85
2024-11-19T00:07:00,839 DEBUG [RS:1;9c732c6543ff:42069 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-19T00:07:00,842 DEBUG [RS:1;9c732c6543ff:42069 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-19T00:07:00,842 DEBUG [RS:1;9c732c6543ff:42069 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-19T00:07:00,844 DEBUG [RS:1;9c732c6543ff:42069 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-19T00:07:00,845 DEBUG [RS:1;9c732c6543ff:42069 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d8313e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0
2024-11-19T00:07:00,858 DEBUG [RS:1;9c732c6543ff:42069 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9c732c6543ff:42069
2024-11-19T00:07:00,859 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-19T00:07:00,859 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-19T00:07:00,859 DEBUG [RS:1;9c732c6543ff:42069 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-19T00:07:00,860 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.HRegionServer(2659): reportForDuty to master=9c732c6543ff,46287,1731974819797 with port=42069, startcode=1731974820802
2024-11-19T00:07:00,860 DEBUG [RS:1;9c732c6543ff:42069 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-19T00:07:00,862 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55419, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-11-19T00:07:00,863 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46287 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9c732c6543ff,42069,1731974820802
2024-11-19T00:07:00,863 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46287 {}] master.ServerManager(517): Registering regionserver=9c732c6543ff,42069,1731974820802
2024-11-19T00:07:00,865 DEBUG [RS:1;9c732c6543ff:42069 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35
2024-11-19T00:07:00,865 DEBUG [RS:1;9c732c6543ff:42069 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43835
2024-11-19T00:07:00,865 DEBUG [RS:1;9c732c6543ff:42069 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-19T00:07:00,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T00:07:00,868 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9c732c6543ff,42069,1731974820802]
2024-11-19T00:07:00,869 DEBUG [RS:1;9c732c6543ff:42069 {}] zookeeper.ZKUtil(111): regionserver:42069-0x1010851646e0002, quorum=127.0.0.1:49660, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9c732c6543ff,42069,1731974820802
2024-11-19T00:07:00,869 WARN  [RS:1;9c732c6543ff:42069 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T00:07:00,869 INFO  [RS:1;9c732c6543ff:42069 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:07:00,869 DEBUG [RS:1;9c732c6543ff:42069 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802
2024-11-19T00:07:00,876 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-19T00:07:00,917 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-19T00:07:00,933 INFO  [RS:1;9c732c6543ff:42069 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T00:07:00,933 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,937 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-19T00:07:00,938 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-19T00:07:00,939 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,939 DEBUG [RS:1;9c732c6543ff:42069 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,939 DEBUG [RS:1;9c732c6543ff:42069 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,939 DEBUG [RS:1;9c732c6543ff:42069 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,939 DEBUG [RS:1;9c732c6543ff:42069 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,939 DEBUG [RS:1;9c732c6543ff:42069 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,939 DEBUG [RS:1;9c732c6543ff:42069 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2
2024-11-19T00:07:00,939 DEBUG [RS:1;9c732c6543ff:42069 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,939 DEBUG [RS:1;9c732c6543ff:42069 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,939 DEBUG [RS:1;9c732c6543ff:42069 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,939 DEBUG [RS:1;9c732c6543ff:42069 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,939 DEBUG [RS:1;9c732c6543ff:42069 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,940 DEBUG [RS:1;9c732c6543ff:42069 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:00,940 DEBUG [RS:1;9c732c6543ff:42069 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3
2024-11-19T00:07:00,940 DEBUG [RS:1;9c732c6543ff:42069 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3
2024-11-19T00:07:00,941 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,941 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,941 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,941 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,941 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,941 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,42069,1731974820802-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T00:07:00,962 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-19T00:07:00,962 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,42069,1731974820802-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,962 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,962 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.Replication(171): 9c732c6543ff,42069,1731974820802 started
2024-11-19T00:07:00,982 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:00,982 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.HRegionServer(1482): Serving as 9c732c6543ff,42069,1731974820802, RpcServer on 9c732c6543ff/172.17.0.2:42069, sessionid=0x1010851646e0002
2024-11-19T00:07:00,982 DEBUG [RS:1;9c732c6543ff:42069 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-19T00:07:00,982 DEBUG [RS:1;9c732c6543ff:42069 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9c732c6543ff,42069,1731974820802
2024-11-19T00:07:00,982 DEBUG [RS:1;9c732c6543ff:42069 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,42069,1731974820802'
2024-11-19T00:07:00,982 DEBUG [RS:1;9c732c6543ff:42069 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-19T00:07:00,982 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;9c732c6543ff:42069,5,FailOnTimeoutGroup]
2024-11-19T00:07:00,983 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-11-19T00:07:00,983 DEBUG [RS:1;9c732c6543ff:42069 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-19T00:07:00,983 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-19T00:07:00,983 DEBUG [RS:1;9c732c6543ff:42069 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-19T00:07:00,983 DEBUG [RS:1;9c732c6543ff:42069 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-19T00:07:00,983 DEBUG [RS:1;9c732c6543ff:42069 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9c732c6543ff,42069,1731974820802
2024-11-19T00:07:00,983 DEBUG [RS:1;9c732c6543ff:42069 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,42069,1731974820802'
2024-11-19T00:07:00,983 DEBUG [RS:1;9c732c6543ff:42069 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-19T00:07:00,984 DEBUG [RS:1;9c732c6543ff:42069 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-19T00:07:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is 9c732c6543ff,46287,1731974819797
2024-11-19T00:07:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7a158168
2024-11-19T00:07:00,985 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-19T00:07:00,985 DEBUG [RS:1;9c732c6543ff:42069 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-19T00:07:00,985 INFO  [RS:1;9c732c6543ff:42069 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-19T00:07:00,985 INFO  [RS:1;9c732c6543ff:42069 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-19T00:07:00,987 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-19T00:07:00,988 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46287 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-19T00:07:00,988 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46287 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-19T00:07:00,988 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46287 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T00:07:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46287 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-11-19T00:07:00,992 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-11-19T00:07:00,992 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:00,992 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46287 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-11-19T00:07:00,994 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-19T00:07:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46287 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T00:07:01,007 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42365 is added to blk_1073741835_1011 (size=393)
2024-11-19T00:07:01,007 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40107 is added to blk_1073741835_1011 (size=393)
2024-11-19T00:07:01,010 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7c7d4ca27e4c223b1425b85ace7933e5, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35
2024-11-19T00:07:01,018 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40107 is added to blk_1073741836_1012 (size=76)
2024-11-19T00:07:01,018 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42365 is added to blk_1073741836_1012 (size=76)
2024-11-19T00:07:01,019 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:07:01,019 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 7c7d4ca27e4c223b1425b85ace7933e5, disabling compactions & flushes
2024-11-19T00:07:01,019 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.
2024-11-19T00:07:01,019 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.
2024-11-19T00:07:01,019 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5. after waiting 0 ms
2024-11-19T00:07:01,019 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.
2024-11-19T00:07:01,019 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.
2024-11-19T00:07:01,019 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7c7d4ca27e4c223b1425b85ace7933e5:
Waiting for close lock at 1731974821019Disabling compacts and flushes for region at 1731974821019Disabling writes for close at 1731974821019Writing region close event to WAL at 1731974821019Closed at 1731974821019
2024-11-19T00:07:01,021 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-11-19T00:07:01,022 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731974821021"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731974821021"}]},"ts":"1731974821021"}
2024-11-19T00:07:01,025 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-19T00:07:01,026 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-19T00:07:01,026 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731974821026"}]},"ts":"1731974821026"}
2024-11-19T00:07:01,029 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-11-19T00:07:01,029 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=7c7d4ca27e4c223b1425b85ace7933e5, ASSIGN}]
2024-11-19T00:07:01,031 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=7c7d4ca27e4c223b1425b85ace7933e5, ASSIGN
2024-11-19T00:07:01,032 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=7c7d4ca27e4c223b1425b85ace7933e5, ASSIGN; state=OFFLINE, location=9c732c6543ff,42145,1731974819848; forceNewPlan=false, retain=false
2024-11-19T00:07:01,087 INFO  [RS:1;9c732c6543ff:42069 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C42069%2C1731974820802, suffix=, logDir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802, archiveDir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/oldWALs, maxLogs=32
2024-11-19T00:07:01,088 INFO  [RS:1;9c732c6543ff:42069 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C42069%2C1731974820802.1731974821088
2024-11-19T00:07:01,095 INFO  [RS:1;9c732c6543ff:42069 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
2024-11-19T00:07:01,096 DEBUG [RS:1;9c732c6543ff:42069 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38697:38697),(127.0.0.1/127.0.0.1:45363:45363)]
2024-11-19T00:07:01,183 INFO  [9c732c6543ff:46287 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-11-19T00:07:01,183 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7c7d4ca27e4c223b1425b85ace7933e5, regionState=OPENING, regionLocation=9c732c6543ff,42145,1731974819848
2024-11-19T00:07:01,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=7c7d4ca27e4c223b1425b85ace7933e5, ASSIGN because future has completed
2024-11-19T00:07:01,187 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c7d4ca27e4c223b1425b85ace7933e5, server=9c732c6543ff,42145,1731974819848}]
2024-11-19T00:07:01,247 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:01,253 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:01,349 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.
2024-11-19T00:07:01,350 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c7d4ca27e4c223b1425b85ace7933e5, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:07:01,350 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:01,350 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:07:01,351 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:01,351 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:01,352 INFO  [StoreOpener-7c7d4ca27e4c223b1425b85ace7933e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 7c7d4ca27e4c223b1425b85ace7933e5 
2024-11-19T00:07:01,354 INFO  [StoreOpener-7c7d4ca27e4c223b1425b85ace7933e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c7d4ca27e4c223b1425b85ace7933e5 columnFamilyName info
2024-11-19T00:07:01,354 DEBUG [StoreOpener-7c7d4ca27e4c223b1425b85ace7933e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:01,355 INFO  [StoreOpener-7c7d4ca27e4c223b1425b85ace7933e5-1 {}] regionserver.HStore(327): Store=7c7d4ca27e4c223b1425b85ace7933e5/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:07:01,355 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:01,356 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:01,356 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:01,357 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:01,357 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:01,360 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:01,369 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:07:01,369 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7c7d4ca27e4c223b1425b85ace7933e5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=821164, jitterRate=0.04416513442993164}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-19T00:07:01,369 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:01,370 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7c7d4ca27e4c223b1425b85ace7933e5:
Running coprocessor pre-open hook at 1731974821351Writing region info on filesystem at 1731974821351Initializing all the Stores at 1731974821352 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974821352Cleaning up temporary data from old regions at 1731974821357 (+5 ms)Running coprocessor post-open hooks at 1731974821369 (+12 ms)Region opened successfully at 1731974821370 (+1 ms)
2024-11-19T00:07:01,371 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5., pid=6, masterSystemTime=1731974821340
2024-11-19T00:07:01,374 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.
2024-11-19T00:07:01,374 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.
2024-11-19T00:07:01,375 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7c7d4ca27e4c223b1425b85ace7933e5, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,42145,1731974819848
2024-11-19T00:07:01,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c7d4ca27e4c223b1425b85ace7933e5, server=9c732c6543ff,42145,1731974819848 because future has completed
2024-11-19T00:07:01,380 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46287 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=9c732c6543ff,42145,1731974819848, table=TestLogRolling-testLogRollOnDatanodeDeath, region=7c7d4ca27e4c223b1425b85ace7933e5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-19T00:07:01,384 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-19T00:07:01,384 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c7d4ca27e4c223b1425b85ace7933e5, server=9c732c6543ff,42145,1731974819848 in 193 msec
2024-11-19T00:07:01,394 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-19T00:07:01,394 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=7c7d4ca27e4c223b1425b85ace7933e5, ASSIGN in 355 msec
2024-11-19T00:07:01,395 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-19T00:07:01,395 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731974821395"}]},"ts":"1731974821395"}
2024-11-19T00:07:01,398 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-11-19T00:07:01,399 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-11-19T00:07:01,402 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 411 msec
2024-11-19T00:07:01,767 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T00:07:01,769 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:01,789 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:01,790 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:01,791 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:06,100 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-11-19T00:07:06,253 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-19T00:07:06,253 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-11-19T00:07:06,852 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T00:07:06,854 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:06,877 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:06,883 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:06,883 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46287 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T00:07:11,004 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-11-19T00:07:11,004 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-11-19T00:07:11,007 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-11-19T00:07:11,007 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.
2024-11-19T00:07:11,029 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:07:11,034 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:07:11,035 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:07:11,035 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:07:11,035 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T00:07:11,036 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75255721{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:07:11,037 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@132d95f4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:07:11,167 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@65266587{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/java.io.tmpdir/jetty-localhost-44325-hadoop-hdfs-3_4_1-tests_jar-_-any-13717143056376903249/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:07:11,167 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2e157477{HTTP/1.1, (http/1.1)}{localhost:44325}
2024-11-19T00:07:11,167 INFO  [Time-limited test {}] server.Server(415): Started @110175ms
2024-11-19T00:07:11,169 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:07:11,203 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:07:11,207 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:07:11,209 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:07:11,209 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:07:11,209 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T00:07:11,212 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fbc343d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:07:11,213 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3986ff43{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:07:11,260 WARN  [Thread-821 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data5/current/BP-1245155782-172.17.0.2-1731974819103/current, will proceed with Du for space computation calculation, 
2024-11-19T00:07:11,260 WARN  [Thread-822 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data6/current/BP-1245155782-172.17.0.2-1731974819103/current, will proceed with Du for space computation calculation, 
2024-11-19T00:07:11,285 WARN  [Thread-801 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:07:11,287 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7a69a2e9ddc696ee with lease ID 0x2cc8ef2c933a09cd: Processing first storage report for DS-f7a5953f-ad89-43e6-a049-90082b0ad605 from datanode DatanodeRegistration(127.0.0.1:38505, datanodeUuid=5caca6f9-2b6a-4c8d-abc1-105c609479e7, infoPort=42967, infoSecurePort=0, ipcPort=45721, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103)
2024-11-19T00:07:11,287 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7a69a2e9ddc696ee with lease ID 0x2cc8ef2c933a09cd: from storage DS-f7a5953f-ad89-43e6-a049-90082b0ad605 node DatanodeRegistration(127.0.0.1:38505, datanodeUuid=5caca6f9-2b6a-4c8d-abc1-105c609479e7, infoPort=42967, infoSecurePort=0, ipcPort=45721, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:07:11,288 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7a69a2e9ddc696ee with lease ID 0x2cc8ef2c933a09cd: Processing first storage report for DS-bd57bf58-d955-48ec-9fdf-3f4f39ba6699 from datanode DatanodeRegistration(127.0.0.1:38505, datanodeUuid=5caca6f9-2b6a-4c8d-abc1-105c609479e7, infoPort=42967, infoSecurePort=0, ipcPort=45721, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103)
2024-11-19T00:07:11,288 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7a69a2e9ddc696ee with lease ID 0x2cc8ef2c933a09cd: from storage DS-bd57bf58-d955-48ec-9fdf-3f4f39ba6699 node DatanodeRegistration(127.0.0.1:38505, datanodeUuid=5caca6f9-2b6a-4c8d-abc1-105c609479e7, infoPort=42967, infoSecurePort=0, ipcPort=45721, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:07:11,336 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@458e5bcf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/java.io.tmpdir/jetty-localhost-38475-hadoop-hdfs-3_4_1-tests_jar-_-any-13278798287029330824/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:07:11,336 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6b75ba45{HTTP/1.1, (http/1.1)}{localhost:38475}
2024-11-19T00:07:11,336 INFO  [Time-limited test {}] server.Server(415): Started @110343ms
2024-11-19T00:07:11,337 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:07:11,377 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:07:11,381 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:07:11,383 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:07:11,383 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:07:11,383 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T00:07:11,384 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@495a6aea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:07:11,384 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@15751333{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:07:11,421 WARN  [Thread-856 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data7/current/BP-1245155782-172.17.0.2-1731974819103/current, will proceed with Du for space computation calculation, 
2024-11-19T00:07:11,421 WARN  [Thread-857 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data8/current/BP-1245155782-172.17.0.2-1731974819103/current, will proceed with Du for space computation calculation, 
2024-11-19T00:07:11,444 WARN  [Thread-836 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:07:11,446 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2a92ebd1f66aacb with lease ID 0x2cc8ef2c933a09ce: Processing first storage report for DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4 from datanode DatanodeRegistration(127.0.0.1:35029, datanodeUuid=c3327d25-7eb4-4415-97da-c47996dcaff3, infoPort=45405, infoSecurePort=0, ipcPort=43605, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103)
2024-11-19T00:07:11,446 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a92ebd1f66aacb with lease ID 0x2cc8ef2c933a09ce: from storage DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4 node DatanodeRegistration(127.0.0.1:35029, datanodeUuid=c3327d25-7eb4-4415-97da-c47996dcaff3, infoPort=45405, infoSecurePort=0, ipcPort=43605, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:07:11,446 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2a92ebd1f66aacb with lease ID 0x2cc8ef2c933a09ce: Processing first storage report for DS-a0667b67-0681-44a8-89d2-c6b72ac07da5 from datanode DatanodeRegistration(127.0.0.1:35029, datanodeUuid=c3327d25-7eb4-4415-97da-c47996dcaff3, infoPort=45405, infoSecurePort=0, ipcPort=43605, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103)
2024-11-19T00:07:11,446 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a92ebd1f66aacb with lease ID 0x2cc8ef2c933a09ce: from storage DS-a0667b67-0681-44a8-89d2-c6b72ac07da5 node DatanodeRegistration(127.0.0.1:35029, datanodeUuid=c3327d25-7eb4-4415-97da-c47996dcaff3, infoPort=45405, infoSecurePort=0, ipcPort=43605, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:07:11,508 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@47d01054{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/java.io.tmpdir/jetty-localhost-42837-hadoop-hdfs-3_4_1-tests_jar-_-any-12632318851899993719/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:07:11,508 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17900346{HTTP/1.1, (http/1.1)}{localhost:42837}
2024-11-19T00:07:11,509 INFO  [Time-limited test {}] server.Server(415): Started @110516ms
2024-11-19T00:07:11,510 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:07:11,589 WARN  [Thread-883 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data10/current/BP-1245155782-172.17.0.2-1731974819103/current, will proceed with Du for space computation calculation, 
2024-11-19T00:07:11,589 WARN  [Thread-882 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data9/current/BP-1245155782-172.17.0.2-1731974819103/current, will proceed with Du for space computation calculation, 
2024-11-19T00:07:11,606 WARN  [Thread-871 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:07:11,608 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd2abdcc11a2bb60d with lease ID 0x2cc8ef2c933a09cf: Processing first storage report for DS-b692e405-1e4d-4f68-9d27-eaee00fbe866 from datanode DatanodeRegistration(127.0.0.1:44795, datanodeUuid=1579a991-4d91-4989-8560-4805a012facd, infoPort=43425, infoSecurePort=0, ipcPort=46241, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103)
2024-11-19T00:07:11,608 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd2abdcc11a2bb60d with lease ID 0x2cc8ef2c933a09cf: from storage DS-b692e405-1e4d-4f68-9d27-eaee00fbe866 node DatanodeRegistration(127.0.0.1:44795, datanodeUuid=1579a991-4d91-4989-8560-4805a012facd, infoPort=43425, infoSecurePort=0, ipcPort=46241, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:07:11,608 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd2abdcc11a2bb60d with lease ID 0x2cc8ef2c933a09cf: Processing first storage report for DS-e12faea3-6fca-4b9d-9abc-9a8bd712148a from datanode DatanodeRegistration(127.0.0.1:44795, datanodeUuid=1579a991-4d91-4989-8560-4805a012facd, infoPort=43425, infoSecurePort=0, ipcPort=46241, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103)
2024-11-19T00:07:11,608 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd2abdcc11a2bb60d with lease ID 0x2cc8ef2c933a09cf: from storage DS-e12faea3-6fca-4b9d-9abc-9a8bd712148a node DatanodeRegistration(127.0.0.1:44795, datanodeUuid=1579a991-4d91-4989-8560-4805a012facd, infoPort=43425, infoSecurePort=0, ipcPort=46241, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:07:11,631 WARN  [ResponseProcessor for block BP-1245155782-172.17.0.2-1731974819103:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1245155782-172.17.0.2-1731974819103:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-1245155782-172.17.0.2-1731974819103:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:11,631 WARN  [ResponseProcessor for block BP-1245155782-172.17.0.2-1731974819103:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1245155782-172.17.0.2-1731974819103:blk_1073741837_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:11,631 WARN  [ResponseProcessor for block BP-1245155782-172.17.0.2-1731974819103:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1245155782-172.17.0.2-1731974819103:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:11,631 WARN  [ResponseProcessor for block BP-1245155782-172.17.0.2-1731974819103:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1245155782-172.17.0.2-1731974819103:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-1245155782-172.17.0.2-1731974819103:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:11,632 WARN  [DataStreamer for file /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta block BP-1245155782-172.17.0.2-1731974819103:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK], DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]) is bad.
2024-11-19T00:07:11,631 WARN  [DataStreamer for file /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088 block BP-1245155782-172.17.0.2-1731974819103:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK], DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]) is bad.
2024-11-19T00:07:11,632 WARN  [DataStreamer for file /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/WALs/9c732c6543ff,46287,1731974819797/9c732c6543ff%2C46287%2C1731974819797.1731974819946 block BP-1245155782-172.17.0.2-1731974819103:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK], DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]) is bad.
2024-11-19T00:07:11,631 WARN  [DataStreamer for file /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245 block BP-1245155782-172.17.0.2-1731974819103:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK], DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]) is bad.
2024-11-19T00:07:11,632 WARN  [PacketResponder: BP-1245155782-172.17.0.2-1731974819103:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:40107] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:11,633 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-213338104_22 at /127.0.0.1:49560 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:40107:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49560 dst: /127.0.0.1:40107
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:11,633 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-213338104_22 at /127.0.0.1:48042 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:42365:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48042 dst: /127.0.0.1:42365
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:11,633 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:48020 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:42365:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48020 dst: /127.0.0.1:42365
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:11,633 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1568943000_22 at /127.0.0.1:49502 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:40107:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49502 dst: /127.0.0.1:40107
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:11,633 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:49524 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:40107:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49524 dst: /127.0.0.1:40107
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:11,633 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:48026 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:42365:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48026 dst: /127.0.0.1:42365
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:11,633 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1568943000_22 at /127.0.0.1:47990 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:42365:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47990 dst: /127.0.0.1:42365
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:11,634 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:49528 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:40107:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49528 dst: /127.0.0.1:40107
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:11,636 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@78ab676e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:07:11,637 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4478d7de{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:07:11,637 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:07:11,637 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fe8f7b0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:07:11,637 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6471b09b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.log.dir/,STOPPED}
2024-11-19T00:07:11,638 WARN  [BP-1245155782-172.17.0.2-1731974819103 heartbeating to localhost/127.0.0.1:43835 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:07:11,638 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:07:11,638 WARN  [BP-1245155782-172.17.0.2-1731974819103 heartbeating to localhost/127.0.0.1:43835 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1245155782-172.17.0.2-1731974819103 (Datanode Uuid 3f4a546e-1f94-4f62-a530-2c54092a62e8) service to localhost/127.0.0.1:43835
2024-11-19T00:07:11,638 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:07:11,639 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data3/current/BP-1245155782-172.17.0.2-1731974819103 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:07:11,639 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data4/current/BP-1245155782-172.17.0.2-1731974819103 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:07:11,639 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:07:11,639 WARN  [DataStreamer for file /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088 block BP-1245155782-172.17.0.2-1731974819103:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:11,643 WARN  [DataStreamer for file /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta block BP-1245155782-172.17.0.2-1731974819103:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:11,643 WARN  [DataStreamer for file /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245 block BP-1245155782-172.17.0.2-1731974819103:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:11,644 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@342ceee3 {}] datanode.DataXceiver(331): 127.0.0.1:42365:DataXceiver error processing unknown operation  src: /127.0.0.1:59032 dst: /127.0.0.1:42365
java.io.IOException: Server closed.
	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:11,645 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3150e6db{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:07:11,645 WARN  [DataStreamer for file /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/WALs/9c732c6543ff,46287,1731974819797/9c732c6543ff%2C46287%2C1731974819797.1731974819946 block BP-1245155782-172.17.0.2-1731974819103:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:11,645 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@539e83f9{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:07:11,645 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:07:11,645 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@259c861e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:07:11,645 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5917cb43{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.log.dir/,STOPPED}
2024-11-19T00:07:11,647 WARN  [BP-1245155782-172.17.0.2-1731974819103 heartbeating to localhost/127.0.0.1:43835 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:07:11,647 WARN  [BP-1245155782-172.17.0.2-1731974819103 heartbeating to localhost/127.0.0.1:43835 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1245155782-172.17.0.2-1731974819103 (Datanode Uuid 5986ea1c-b94a-4f7a-ba8d-0f5e20139946) service to localhost/127.0.0.1:43835
2024-11-19T00:07:11,647 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data1/current/BP-1245155782-172.17.0.2-1731974819103 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:07:11,647 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data2/current/BP-1245155782-172.17.0.2-1731974819103 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:07:11,647 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:07:11,647 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:07:11,647 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:07:11,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5., hostname=9c732c6543ff,42145,1731974819848, seqNum=2]
2024-11-19T00:07:11,653 ERROR [FSHLog-0-hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35-prefix:9c732c6543ff,42145,1731974819848 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:11,653 WARN  [FSHLog-0-hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35-prefix:9c732c6543ff,42145,1731974819848 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:11,653 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:11,653 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9c732c6543ff%2C42145%2C1731974819848:(num 1731974820245) roll requested
2024-11-19T00:07:11,654 INFO  [regionserver/9c732c6543ff:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C42145%2C1731974819848.1731974831653
2024-11-19T00:07:11,661 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:11,661 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:11,661 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:11,662 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:11,662 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:11,662 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974831653
2024-11-19T00:07:11,662 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:11,662 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:11,663 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43425:43425),(127.0.0.1/127.0.0.1:42967:42967)]
2024-11-19T00:07:11,663 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245 is not closed yet, will try archiving it next time
2024-11-19T00:07:11,664 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-11-19T00:07:11,664 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-11-19T00:07:11,664 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
2024-11-19T00:07:11,667 WARN  [IPC Server handler 1 on default port 43835 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1009
2024-11-19T00:07:11,671 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245 after 5ms
2024-11-19T00:07:11,926 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:12,941 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:13,663 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:13,664 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974831653
2024-11-19T00:07:13,665 WARN  [ResponseProcessor for block BP-1245155782-172.17.0.2-1731974819103:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1245155782-172.17.0.2-1731974819103:blk_1073741838_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:13,665 WARN  [DataStreamer for file /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974831653 block BP-1245155782-172.17.0.2-1731974819103:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK], DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:13,666 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:51970 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:44795:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51970 dst: /127.0.0.1:44795
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:13,666 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:33578 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:38505:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33578 dst: /127.0.0.1:38505
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:13,670 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@47d01054{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:07:13,671 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17900346{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:07:13,671 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:07:13,671 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@15751333{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:07:13,671 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@495a6aea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.log.dir/,STOPPED}
2024-11-19T00:07:13,673 WARN  [BP-1245155782-172.17.0.2-1731974819103 heartbeating to localhost/127.0.0.1:43835 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:07:13,673 WARN  [BP-1245155782-172.17.0.2-1731974819103 heartbeating to localhost/127.0.0.1:43835 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1245155782-172.17.0.2-1731974819103 (Datanode Uuid 1579a991-4d91-4989-8560-4805a012facd) service to localhost/127.0.0.1:43835
2024-11-19T00:07:13,674 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data9/current/BP-1245155782-172.17.0.2-1731974819103 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:07:13,674 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data10/current/BP-1245155782-172.17.0.2-1731974819103 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:07:13,674 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:07:13,674 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:07:13,674 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:07:13,926 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:14,941 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:15,664 WARN  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]]
2024-11-19T00:07:15,664 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:15,664 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9c732c6543ff%2C42145%2C1731974819848:(num 1731974831653) roll requested
2024-11-19T00:07:15,664 INFO  [regionserver/9c732c6543ff:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C42145%2C1731974819848.1731974835664
2024-11-19T00:07:15,667 WARN  [Thread-903 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1021
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:15,668 WARN  [Thread-903 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK], DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:15,668 WARN  [Thread-903 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741839_1021
2024-11-19T00:07:15,671 WARN  [Thread-903 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]
2024-11-19T00:07:15,672 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245 after 4008ms
2024-11-19T00:07:15,679 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:15,679 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:15,679 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:15,679 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:15,679 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:15,680 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974831653 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974835664
2024-11-19T00:07:15,681 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-19T00:07:15,683 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38505 is added to blk_1073741838_1020 (size=2431)
2024-11-19T00:07:15,685 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42967:42967),(127.0.0.1/127.0.0.1:45405:45405)]
2024-11-19T00:07:15,686 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245 is not closed yet, will try archiving it next time
2024-11-19T00:07:15,686 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974831653 is not closed yet, will try archiving it next time
2024-11-19T00:07:15,926 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:16,084 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245 is not closed yet, will try archiving it next time
2024-11-19T00:07:16,942 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:17,686 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:17,687 WARN  [ResponseProcessor for block BP-1245155782-172.17.0.2-1731974819103:blk_1073741840_1022 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1245155782-172.17.0.2-1731974819103:blk_1073741840_1022
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:17,688 WARN  [DataStreamer for file /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974835664 block BP-1245155782-172.17.0.2-1731974819103:blk_1073741840_1022 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]) is bad.
2024-11-19T00:07:17,688 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:53096 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:38505:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53096 dst: /127.0.0.1:38505
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:17,689 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35640 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:35029:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35640 dst: /127.0.0.1:35029
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:17,691 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@65266587{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:07:17,692 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2e157477{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:07:17,692 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:07:17,692 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@132d95f4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:07:17,692 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75255721{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.log.dir/,STOPPED}
2024-11-19T00:07:17,694 WARN  [BP-1245155782-172.17.0.2-1731974819103 heartbeating to localhost/127.0.0.1:43835 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:07:17,694 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:07:17,694 WARN  [BP-1245155782-172.17.0.2-1731974819103 heartbeating to localhost/127.0.0.1:43835 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1245155782-172.17.0.2-1731974819103 (Datanode Uuid 5caca6f9-2b6a-4c8d-abc1-105c609479e7) service to localhost/127.0.0.1:43835
2024-11-19T00:07:17,694 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:07:17,694 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data5/current/BP-1245155782-172.17.0.2-1731974819103 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:07:17,695 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data6/current/BP-1245155782-172.17.0.2-1731974819103 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:07:17,695 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:07:17,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42145 {}] regionserver.HRegion(8855): Flush requested on 7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:17,706 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7d4ca27e4c223b1425b85ace7933e5 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T00:07:17,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/6ebdd5c2b89546ebbab5c2e4086ac36c is 1080, key is row0002/info:/1731974833676/Put/seqid=0
2024-11-19T00:07:17,727 WARN  [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1024
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:17,728 WARN  [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741841_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK], DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]) is bad.
2024-11-19T00:07:17,728 WARN  [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741841_1024
2024-11-19T00:07:17,728 WARN  [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]
2024-11-19T00:07:17,730 WARN  [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1025
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:17,730 WARN  [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK], DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]) is bad.
2024-11-19T00:07:17,730 WARN  [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741842_1025
2024-11-19T00:07:17,731 WARN  [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]
2024-11-19T00:07:17,733 WARN  [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44795
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:17,733 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35660 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741843_1026] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data8]'}, localName='127.0.0.1:35029', datanodeUuid='c3327d25-7eb4-4415-97da-c47996dcaff3', xmitsInProgress=0}:Exception transferring block BP-1245155782-172.17.0.2-1731974819103:blk_1073741843_1026 to mirror 127.0.0.1:44795
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:17,733 WARN  [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK], DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:17,733 WARN  [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741843_1026
2024-11-19T00:07:17,734 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35660 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741843_1026] {}] datanode.BlockReceiver(316): Block 1073741843 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T00:07:17,734 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35660 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741843_1026] {}] datanode.DataXceiver(331): 127.0.0.1:35029:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35660 dst: /127.0.0.1:35029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:17,734 WARN  [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]
2024-11-19T00:07:17,736 WARN  [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38505
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:17,736 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35676 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741844_1027] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data8]'}, localName='127.0.0.1:35029', datanodeUuid='c3327d25-7eb4-4415-97da-c47996dcaff3', xmitsInProgress=0}:Exception transferring block BP-1245155782-172.17.0.2-1731974819103:blk_1073741844_1027 to mirror 127.0.0.1:38505
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:17,736 WARN  [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK], DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]) is bad.
2024-11-19T00:07:17,736 WARN  [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741844_1027
2024-11-19T00:07:17,736 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35676 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741844_1027] {}] datanode.BlockReceiver(316): Block 1073741844 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T00:07:17,737 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35676 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741844_1027] {}] datanode.DataXceiver(331): 127.0.0.1:35029:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35676 dst: /127.0.0.1:35029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:17,737 WARN  [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]
2024-11-19T00:07:17,738 WARN  [IPC Server handler 1 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T00:07:17,738 WARN  [IPC Server handler 1 on default port 43835 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T00:07:17,738 WARN  [IPC Server handler 1 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T00:07:17,743 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741845_1028 (size=10347)
2024-11-19T00:07:17,927 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:18,144 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/6ebdd5c2b89546ebbab5c2e4086ac36c
2024-11-19T00:07:18,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/6ebdd5c2b89546ebbab5c2e4086ac36c as hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/6ebdd5c2b89546ebbab5c2e4086ac36c
2024-11-19T00:07:18,160 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/6ebdd5c2b89546ebbab5c2e4086ac36c, entries=5, sequenceid=11, filesize=10.1 K
2024-11-19T00:07:18,161 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 7c7d4ca27e4c223b1425b85ace7933e5 in 455ms, sequenceid=11, compaction requested=false
2024-11-19T00:07:18,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7d4ca27e4c223b1425b85ace7933e5:

2024-11-19T00:07:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42145 {}] regionserver.HRegion(8855): Flush requested on 7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:18,332 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7d4ca27e4c223b1425b85ace7933e5 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-11-19T00:07:18,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/96748ab60dd541d38a53ac3eced4fc48 is 1080, key is row0007/info:/1731974837707/Put/seqid=0
2024-11-19T00:07:18,339 WARN  [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:18,339 WARN  [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK], DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:18,339 WARN  [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741846_1029
2024-11-19T00:07:18,340 WARN  [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]
2024-11-19T00:07:18,341 WARN  [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:18,341 WARN  [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK], DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]) is bad.
2024-11-19T00:07:18,341 WARN  [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741847_1030
2024-11-19T00:07:18,342 WARN  [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]
2024-11-19T00:07:18,344 WARN  [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42365
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:18,344 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35694 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741848_1031] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data8]'}, localName='127.0.0.1:35029', datanodeUuid='c3327d25-7eb4-4415-97da-c47996dcaff3', xmitsInProgress=0}:Exception transferring block BP-1245155782-172.17.0.2-1731974819103:blk_1073741848_1031 to mirror 127.0.0.1:42365
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:18,345 WARN  [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK], DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]) is bad.
2024-11-19T00:07:18,345 WARN  [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741848_1031
2024-11-19T00:07:18,345 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35694 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741848_1031] {}] datanode.BlockReceiver(316): Block 1073741848 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T00:07:18,345 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35694 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741848_1031] {}] datanode.DataXceiver(331): 127.0.0.1:35029:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35694 dst: /127.0.0.1:35029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:18,345 WARN  [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]
2024-11-19T00:07:18,346 WARN  [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:18,346 WARN  [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]) is bad.
2024-11-19T00:07:18,347 WARN  [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741849_1032
2024-11-19T00:07:18,347 WARN  [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]
2024-11-19T00:07:18,347 WARN  [IPC Server handler 1 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T00:07:18,348 WARN  [IPC Server handler 1 on default port 43835 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T00:07:18,348 WARN  [IPC Server handler 1 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T00:07:18,351 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741850_1033 (size=12506)
2024-11-19T00:07:18,352 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/96748ab60dd541d38a53ac3eced4fc48
2024-11-19T00:07:18,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/96748ab60dd541d38a53ac3eced4fc48 as hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/96748ab60dd541d38a53ac3eced4fc48
2024-11-19T00:07:18,366 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/96748ab60dd541d38a53ac3eced4fc48, entries=7, sequenceid=24, filesize=12.2 K
2024-11-19T00:07:18,368 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=0 B/0 for 7c7d4ca27e4c223b1425b85ace7933e5 in 36ms, sequenceid=24, compaction requested=false
2024-11-19T00:07:18,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7d4ca27e4c223b1425b85ace7933e5:

2024-11-19T00:07:18,368 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-11-19T00:07:18,368 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:07:18,368 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/96748ab60dd541d38a53ac3eced4fc48 because midkey is the same as first or last row
2024-11-19T00:07:18,942 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:19,686 WARN  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]
2024-11-19T00:07:19,686 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:19,687 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9c732c6543ff%2C42145%2C1731974819848:(num 1731974835664) roll requested
2024-11-19T00:07:19,687 INFO  [regionserver/9c732c6543ff:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C42145%2C1731974819848.1731974839687
2024-11-19T00:07:19,690 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:19,690 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK], DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]) is bad.
2024-11-19T00:07:19,691 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741851_1034
2024-11-19T00:07:19,691 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]
2024-11-19T00:07:19,694 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44795
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:19,694 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35712 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741852_1035] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data8]'}, localName='127.0.0.1:35029', datanodeUuid='c3327d25-7eb4-4415-97da-c47996dcaff3', xmitsInProgress=0}:Exception transferring block BP-1245155782-172.17.0.2-1731974819103:blk_1073741852_1035 to mirror 127.0.0.1:44795
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:19,694 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK], DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:19,694 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35712 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741852_1035] {}] datanode.BlockReceiver(316): Block 1073741852 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-19T00:07:19,694 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741852_1035
2024-11-19T00:07:19,694 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35712 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741852_1035] {}] datanode.DataXceiver(331): 127.0.0.1:35029:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35712 dst: /127.0.0.1:35029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:19,695 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]
2024-11-19T00:07:19,696 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:19,696 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]) is bad.
2024-11-19T00:07:19,696 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741853_1036
2024-11-19T00:07:19,697 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]
2024-11-19T00:07:19,698 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:19,698 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]) is bad.
2024-11-19T00:07:19,698 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741854_1037
2024-11-19T00:07:19,698 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]
2024-11-19T00:07:19,699 WARN  [IPC Server handler 1 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T00:07:19,699 WARN  [IPC Server handler 1 on default port 43835 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T00:07:19,699 WARN  [IPC Server handler 1 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T00:07:19,702 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:19,702 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:19,702 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:19,702 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:19,702 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:19,703 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974835664 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974839687
2024-11-19T00:07:19,704 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45405:45405)]
2024-11-19T00:07:19,704 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245 is not closed yet, will try archiving it next time
2024-11-19T00:07:19,704 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974835664 is not closed yet, will try archiving it next time
2024-11-19T00:07:19,704 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741840_1023 (size=25992)
2024-11-19T00:07:19,705 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974831653 to hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/oldWALs/9c732c6543ff%2C42145%2C1731974819848.1731974831653
2024-11-19T00:07:19,706 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245 is not closed yet, will try archiving it next time
2024-11-19T00:07:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42145 {}] regionserver.HRegion(8855): Flush requested on 7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:19,752 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7d4ca27e4c223b1425b85ace7933e5 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-19T00:07:19,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/c9f908752b304476a1c03b560f8fb5fb is 1079, key is tmprow/info:/1731974839751/Put/seqid=0
2024-11-19T00:07:19,759 WARN  [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:19,759 WARN  [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:19,759 WARN  [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741856_1039
2024-11-19T00:07:19,760 WARN  [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]
2024-11-19T00:07:19,761 WARN  [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:19,761 WARN  [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK], DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]) is bad.
2024-11-19T00:07:19,762 WARN  [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741857_1040
2024-11-19T00:07:19,762 WARN  [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]
2024-11-19T00:07:19,763 WARN  [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:19,763 WARN  [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]) is bad.
2024-11-19T00:07:19,763 WARN  [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741858_1041
2024-11-19T00:07:19,764 WARN  [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]
2024-11-19T00:07:19,765 WARN  [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:19,765 WARN  [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]) is bad.
2024-11-19T00:07:19,765 WARN  [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741859_1042
2024-11-19T00:07:19,766 WARN  [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]
2024-11-19T00:07:19,766 WARN  [IPC Server handler 0 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T00:07:19,766 WARN  [IPC Server handler 0 on default port 43835 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T00:07:19,766 WARN  [IPC Server handler 0 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T00:07:19,770 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741860_1043 (size=6027)
2024-11-19T00:07:19,927 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:20,171 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/c9f908752b304476a1c03b560f8fb5fb
2024-11-19T00:07:20,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/c9f908752b304476a1c03b560f8fb5fb as hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/c9f908752b304476a1c03b560f8fb5fb
2024-11-19T00:07:20,186 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/c9f908752b304476a1c03b560f8fb5fb, entries=1, sequenceid=34, filesize=5.9 K
2024-11-19T00:07:20,187 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 7c7d4ca27e4c223b1425b85ace7933e5 in 435ms, sequenceid=34, compaction requested=true
2024-11-19T00:07:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7d4ca27e4c223b1425b85ace7933e5:

2024-11-19T00:07:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-11-19T00:07:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:07:20,188 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/96748ab60dd541d38a53ac3eced4fc48 because midkey is the same as first or last row
2024-11-19T00:07:20,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7d4ca27e4c223b1425b85ace7933e5:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T00:07:20,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:07:20,189 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T00:07:20,190 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T00:07:20,190 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HStore(1541): 7c7d4ca27e4c223b1425b85ace7933e5/info is initiating minor compaction (all files)
2024-11-19T00:07:20,190 INFO  [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7d4ca27e4c223b1425b85ace7933e5/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.
2024-11-19T00:07:20,190 INFO  [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/6ebdd5c2b89546ebbab5c2e4086ac36c, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/96748ab60dd541d38a53ac3eced4fc48, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/c9f908752b304476a1c03b560f8fb5fb] into tmpdir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp, totalSize=28.2 K
2024-11-19T00:07:20,191 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ebdd5c2b89546ebbab5c2e4086ac36c, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731974833676
2024-11-19T00:07:20,191 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96748ab60dd541d38a53ac3eced4fc48, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1731974837707
2024-11-19T00:07:20,192 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9f908752b304476a1c03b560f8fb5fb, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731974839751
2024-11-19T00:07:20,209 INFO  [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7d4ca27e4c223b1425b85ace7933e5#info#compaction#21 average throughput is 4.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T00:07:20,209 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/e7e854e9361b458d9eee770570edc73c is 1080, key is row0002/info:/1731974833676/Put/seqid=0
2024-11-19T00:07:20,212 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44795
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:20,212 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35758 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741861_1044] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data8]'}, localName='127.0.0.1:35029', datanodeUuid='c3327d25-7eb4-4415-97da-c47996dcaff3', xmitsInProgress=0}:Exception transferring block BP-1245155782-172.17.0.2-1731974819103:blk_1073741861_1044 to mirror 127.0.0.1:44795
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:20,213 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK], DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:20,213 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741861_1044
2024-11-19T00:07:20,213 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35758 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741861_1044] {}] datanode.BlockReceiver(316): Block 1073741861 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T00:07:20,213 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35758 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741861_1044] {}] datanode.DataXceiver(331): 127.0.0.1:35029:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35758 dst: /127.0.0.1:35029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:20,213 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]
2024-11-19T00:07:20,214 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:20,215 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK], DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]) is bad.
2024-11-19T00:07:20,215 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741862_1045
2024-11-19T00:07:20,215 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]
2024-11-19T00:07:20,216 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:20,216 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]) is bad.
2024-11-19T00:07:20,216 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741863_1046
2024-11-19T00:07:20,217 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]
2024-11-19T00:07:20,218 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:20,218 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]) is bad.
2024-11-19T00:07:20,218 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741864_1047
2024-11-19T00:07:20,219 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]
2024-11-19T00:07:20,219 WARN  [IPC Server handler 2 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T00:07:20,219 WARN  [IPC Server handler 2 on default port 43835 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T00:07:20,219 WARN  [IPC Server handler 2 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T00:07:20,222 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741865_1048 (size=17994)
2024-11-19T00:07:20,460 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@57e35ef0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35029, datanodeUuid=c3327d25-7eb4-4415-97da-c47996dcaff3, infoPort=45405, infoSecurePort=0, ipcPort=43605, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103):Failed to transfer BP-1245155782-172.17.0.2-1731974819103:blk_1073741845_1028 to 127.0.0.1:42365 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:20,460 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@262f9cd[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35029, datanodeUuid=c3327d25-7eb4-4415-97da-c47996dcaff3, infoPort=45405, infoSecurePort=0, ipcPort=43605, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103):Failed to transfer BP-1245155782-172.17.0.2-1731974819103:blk_1073741850_1033 to 127.0.0.1:40107 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:20,631 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/e7e854e9361b458d9eee770570edc73c as hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/e7e854e9361b458d9eee770570edc73c
2024-11-19T00:07:20,638 INFO  [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c7d4ca27e4c223b1425b85ace7933e5/info of 7c7d4ca27e4c223b1425b85ace7933e5 into e7e854e9361b458d9eee770570edc73c(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T00:07:20,638 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7d4ca27e4c223b1425b85ace7933e5:

2024-11-19T00:07:20,638 INFO  [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5., storeName=7c7d4ca27e4c223b1425b85ace7933e5/info, priority=13, startTime=1731974840188; duration=0sec
2024-11-19T00:07:20,639 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-19T00:07:20,639 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:07:20,639 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/e7e854e9361b458d9eee770570edc73c because midkey is the same as first or last row
2024-11-19T00:07:20,639 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-19T00:07:20,639 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:07:20,639 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/e7e854e9361b458d9eee770570edc73c because midkey is the same as first or last row
2024-11-19T00:07:20,639 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-19T00:07:20,639 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:07:20,639 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/e7e854e9361b458d9eee770570edc73c because midkey is the same as first or last row
2024-11-19T00:07:20,639 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:07:20,639 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7d4ca27e4c223b1425b85ace7933e5:info
2024-11-19T00:07:20,942 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:21,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42145 {}] regionserver.HRegion(8855): Flush requested on 7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:21,173 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7d4ca27e4c223b1425b85ace7933e5 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-19T00:07:21,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/af8c5b31d884402494046d88d18b49d3 is 1079, key is tmprow/info:/1731974841171/Put/seqid=0
2024-11-19T00:07:21,179 WARN  [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:21,180 WARN  [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK], DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]) is bad.
2024-11-19T00:07:21,180 WARN  [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741866_1049
2024-11-19T00:07:21,180 WARN  [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]
2024-11-19T00:07:21,181 WARN  [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:21,182 WARN  [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK], DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]) is bad.
2024-11-19T00:07:21,182 WARN  [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741867_1050
2024-11-19T00:07:21,182 WARN  [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]
2024-11-19T00:07:21,183 WARN  [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:21,184 WARN  [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:21,184 WARN  [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741868_1051
2024-11-19T00:07:21,184 WARN  [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]
2024-11-19T00:07:21,186 WARN  [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42365
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:21,186 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35772 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741869_1052] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data8]'}, localName='127.0.0.1:35029', datanodeUuid='c3327d25-7eb4-4415-97da-c47996dcaff3', xmitsInProgress=0}:Exception transferring block BP-1245155782-172.17.0.2-1731974819103:blk_1073741869_1052 to mirror 127.0.0.1:42365
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:21,186 WARN  [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK], DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]) is bad.
2024-11-19T00:07:21,187 WARN  [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741869_1052
2024-11-19T00:07:21,187 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35772 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741869_1052] {}] datanode.BlockReceiver(316): Block 1073741869 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T00:07:21,187 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:35772 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741869_1052] {}] datanode.DataXceiver(331): 127.0.0.1:35029:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35772 dst: /127.0.0.1:35029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:21,187 WARN  [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]
2024-11-19T00:07:21,188 WARN  [IPC Server handler 4 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T00:07:21,188 WARN  [IPC Server handler 4 on default port 43835 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T00:07:21,188 WARN  [IPC Server handler 4 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T00:07:21,191 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741870_1053 (size=6027)
2024-11-19T00:07:21,448 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@57e35ef0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35029, datanodeUuid=c3327d25-7eb4-4415-97da-c47996dcaff3, infoPort=45405, infoSecurePort=0, ipcPort=43605, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103):Failed to transfer BP-1245155782-172.17.0.2-1731974819103:blk_1073741860_1043 to 127.0.0.1:42365 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:21,448 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@262f9cd[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35029, datanodeUuid=c3327d25-7eb4-4415-97da-c47996dcaff3, infoPort=45405, infoSecurePort=0, ipcPort=43605, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103):Failed to transfer BP-1245155782-172.17.0.2-1731974819103:blk_1073741840_1023 to 127.0.0.1:44795 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:21,592 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/af8c5b31d884402494046d88d18b49d3
2024-11-19T00:07:21,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/af8c5b31d884402494046d88d18b49d3 as hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/af8c5b31d884402494046d88d18b49d3
2024-11-19T00:07:21,606 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/af8c5b31d884402494046d88d18b49d3, entries=1, sequenceid=45, filesize=5.9 K
2024-11-19T00:07:21,607 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 7c7d4ca27e4c223b1425b85ace7933e5 in 434ms, sequenceid=45, compaction requested=false
2024-11-19T00:07:21,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7d4ca27e4c223b1425b85ace7933e5:

2024-11-19T00:07:21,608 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-11-19T00:07:21,608 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:07:21,608 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/e7e854e9361b458d9eee770570edc73c because midkey is the same as first or last row
2024-11-19T00:07:21,705 WARN  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]
2024-11-19T00:07:21,705 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:21,705 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9c732c6543ff%2C42145%2C1731974819848:(num 1731974839687) roll requested
2024-11-19T00:07:21,705 INFO  [regionserver/9c732c6543ff:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C42145%2C1731974819848.1731974841705
2024-11-19T00:07:21,708 WARN  [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:21,709 WARN  [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK], DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:21,709 WARN  [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741871_1054
2024-11-19T00:07:21,709 WARN  [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]
2024-11-19T00:07:21,710 WARN  [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:21,711 WARN  [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK], DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]) is bad.
2024-11-19T00:07:21,711 WARN  [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741872_1055
2024-11-19T00:07:21,711 WARN  [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]
2024-11-19T00:07:21,714 WARN  [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38505
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:21,714 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:52316 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741873_1056] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data8]'}, localName='127.0.0.1:35029', datanodeUuid='c3327d25-7eb4-4415-97da-c47996dcaff3', xmitsInProgress=0}:Exception transferring block BP-1245155782-172.17.0.2-1731974819103:blk_1073741873_1056 to mirror 127.0.0.1:38505
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:21,714 WARN  [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK], DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]) is bad.
2024-11-19T00:07:21,714 WARN  [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741873_1056
2024-11-19T00:07:21,714 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:52316 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741873_1056] {}] datanode.BlockReceiver(316): Block 1073741873 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-19T00:07:21,714 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:52316 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741873_1056] {}] datanode.DataXceiver(331): 127.0.0.1:35029:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52316 dst: /127.0.0.1:35029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:21,715 WARN  [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]
2024-11-19T00:07:21,716 WARN  [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:21,716 WARN  [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]) is bad.
2024-11-19T00:07:21,716 WARN  [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741874_1057
2024-11-19T00:07:21,716 WARN  [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]
2024-11-19T00:07:21,717 WARN  [IPC Server handler 4 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T00:07:21,717 WARN  [IPC Server handler 4 on default port 43835 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T00:07:21,717 WARN  [IPC Server handler 4 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T00:07:21,723 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:21,723 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:21,723 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:21,723 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:21,723 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:21,724 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974839687 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974841705
2024-11-19T00:07:21,724 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45405:45405)]
2024-11-19T00:07:21,724 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245 is not closed yet, will try archiving it next time
2024-11-19T00:07:21,724 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974839687 is not closed yet, will try archiving it next time
2024-11-19T00:07:21,725 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974835664 to hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/oldWALs/9c732c6543ff%2C42145%2C1731974819848.1731974835664
2024-11-19T00:07:21,725 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741855_1038 (size=13591)
2024-11-19T00:07:21,928 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:22,126 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245 is not closed yet, will try archiving it next time
2024-11-19T00:07:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42145 {}] regionserver.HRegion(8855): Flush requested on 7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:22,597 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7d4ca27e4c223b1425b85ace7933e5 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-19T00:07:22,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/2ce9a7821392416786ccf9a60bd4b21d is 1079, key is tmprow/info:/1731974842595/Put/seqid=0
2024-11-19T00:07:22,604 WARN  [Thread-954 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:22,604 WARN  [Thread-954 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK], DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:22,604 WARN  [Thread-954 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741876_1059
2024-11-19T00:07:22,605 WARN  [Thread-954 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]
2024-11-19T00:07:22,606 WARN  [Thread-954 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:22,606 WARN  [Thread-954 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK], DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]) is bad.
2024-11-19T00:07:22,606 WARN  [Thread-954 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741877_1060
2024-11-19T00:07:22,606 WARN  [Thread-954 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]
2024-11-19T00:07:22,608 WARN  [Thread-954 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:22,608 WARN  [Thread-954 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]) is bad.
2024-11-19T00:07:22,608 WARN  [Thread-954 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741878_1061
2024-11-19T00:07:22,608 WARN  [Thread-954 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]
2024-11-19T00:07:22,610 WARN  [Thread-954 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:22,610 WARN  [Thread-954 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]) is bad.
2024-11-19T00:07:22,610 WARN  [Thread-954 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741879_1062
2024-11-19T00:07:22,610 WARN  [Thread-954 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]
2024-11-19T00:07:22,611 WARN  [IPC Server handler 1 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T00:07:22,611 WARN  [IPC Server handler 1 on default port 43835 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T00:07:22,611 WARN  [IPC Server handler 1 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T00:07:22,614 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741880_1063 (size=6027)
2024-11-19T00:07:22,943 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:23,015 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/2ce9a7821392416786ccf9a60bd4b21d
2024-11-19T00:07:23,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/2ce9a7821392416786ccf9a60bd4b21d as hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/2ce9a7821392416786ccf9a60bd4b21d
2024-11-19T00:07:23,029 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/2ce9a7821392416786ccf9a60bd4b21d, entries=1, sequenceid=55, filesize=5.9 K
2024-11-19T00:07:23,030 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 7c7d4ca27e4c223b1425b85ace7933e5 in 433ms, sequenceid=55, compaction requested=true
2024-11-19T00:07:23,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7d4ca27e4c223b1425b85ace7933e5:

2024-11-19T00:07:23,030 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-11-19T00:07:23,030 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:07:23,030 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/e7e854e9361b458d9eee770570edc73c because midkey is the same as first or last row
2024-11-19T00:07:23,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7d4ca27e4c223b1425b85ace7933e5:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T00:07:23,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:07:23,030 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T00:07:23,031 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T00:07:23,032 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HStore(1541): 7c7d4ca27e4c223b1425b85ace7933e5/info is initiating minor compaction (all files)
2024-11-19T00:07:23,032 INFO  [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7d4ca27e4c223b1425b85ace7933e5/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.
2024-11-19T00:07:23,032 INFO  [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/e7e854e9361b458d9eee770570edc73c, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/af8c5b31d884402494046d88d18b49d3, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/2ce9a7821392416786ccf9a60bd4b21d] into tmpdir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp, totalSize=29.3 K
2024-11-19T00:07:23,032 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7e854e9361b458d9eee770570edc73c, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731974833676
2024-11-19T00:07:23,033 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] compactions.Compactor(225): Compacting af8c5b31d884402494046d88d18b49d3, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731974841171
2024-11-19T00:07:23,033 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ce9a7821392416786ccf9a60bd4b21d, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731974842595
2024-11-19T00:07:23,049 INFO  [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7d4ca27e4c223b1425b85ace7933e5#info#compaction#24 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T00:07:23,049 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/4d067a1a9bd44af6920af56aa6df5223 is 1080, key is row0002/info:/1731974833676/Put/seqid=0
2024-11-19T00:07:23,051 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:23,051 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]) is bad.
2024-11-19T00:07:23,051 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741881_1064
2024-11-19T00:07:23,052 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]
2024-11-19T00:07:23,053 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:23,054 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]) is bad.
2024-11-19T00:07:23,054 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741882_1065
2024-11-19T00:07:23,054 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40107,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]
2024-11-19T00:07:23,055 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:23,056 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]) is bad.
2024-11-19T00:07:23,056 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741883_1066
2024-11-19T00:07:23,056 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]
2024-11-19T00:07:23,058 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:52352 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741884_1067] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data8]'}, localName='127.0.0.1:35029', datanodeUuid='c3327d25-7eb4-4415-97da-c47996dcaff3', xmitsInProgress=0}:Exception transferring block BP-1245155782-172.17.0.2-1731974819103:blk_1073741884_1067 to mirror 127.0.0.1:44795
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:23,058 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44795
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:23,058 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:52352 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741884_1067] {}] datanode.BlockReceiver(316): Block 1073741884 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T00:07:23,058 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK], DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:23,059 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741884_1067
2024-11-19T00:07:23,059 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:52352 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741884_1067] {}] datanode.DataXceiver(331): 127.0.0.1:35029:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52352 dst: /127.0.0.1:35029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:23,059 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]
2024-11-19T00:07:23,060 WARN  [IPC Server handler 3 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T00:07:23,060 WARN  [IPC Server handler 3 on default port 43835 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T00:07:23,060 WARN  [IPC Server handler 3 on default port 43835 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T00:07:23,063 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741885_1068 (size=18097)
2024-11-19T00:07:23,448 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@57e35ef0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35029, datanodeUuid=c3327d25-7eb4-4415-97da-c47996dcaff3, infoPort=45405, infoSecurePort=0, ipcPort=43605, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103):Failed to transfer BP-1245155782-172.17.0.2-1731974819103:blk_1073741870_1053 to 127.0.0.1:44795 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:23,448 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@262f9cd[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35029, datanodeUuid=c3327d25-7eb4-4415-97da-c47996dcaff3, infoPort=45405, infoSecurePort=0, ipcPort=43605, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103):Failed to transfer BP-1245155782-172.17.0.2-1731974819103:blk_1073741865_1048 to 127.0.0.1:44795 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:23,470 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/4d067a1a9bd44af6920af56aa6df5223 as hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/4d067a1a9bd44af6920af56aa6df5223
2024-11-19T00:07:23,478 INFO  [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c7d4ca27e4c223b1425b85ace7933e5/info of 7c7d4ca27e4c223b1425b85ace7933e5 into 4d067a1a9bd44af6920af56aa6df5223(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T00:07:23,478 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7d4ca27e4c223b1425b85ace7933e5:

2024-11-19T00:07:23,478 INFO  [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5., storeName=7c7d4ca27e4c223b1425b85ace7933e5/info, priority=13, startTime=1731974843030; duration=0sec
2024-11-19T00:07:23,479 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-19T00:07:23,479 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:07:23,479 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/4d067a1a9bd44af6920af56aa6df5223 because midkey is the same as first or last row
2024-11-19T00:07:23,479 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-19T00:07:23,479 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:07:23,479 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/4d067a1a9bd44af6920af56aa6df5223 because midkey is the same as first or last row
2024-11-19T00:07:23,479 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-19T00:07:23,479 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:07:23,479 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/4d067a1a9bd44af6920af56aa6df5223 because midkey is the same as first or last row
2024-11-19T00:07:23,479 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:07:23,479 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7d4ca27e4c223b1425b85ace7933e5:info
2024-11-19T00:07:23,725 WARN  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-11-19T00:07:23,725 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:23,823 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:07:23,826 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:07:23,827 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:07:23,827 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:07:23,827 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T00:07:23,827 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1191c470{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:07:23,828 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69a3dde8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:07:23,928 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:23,947 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@69aa82e5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/java.io.tmpdir/jetty-localhost-32801-hadoop-hdfs-3_4_1-tests_jar-_-any-1337282957929408757/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:07:23,948 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6fc4b42f{HTTP/1.1, (http/1.1)}{localhost:32801}
2024-11-19T00:07:23,948 INFO  [Time-limited test {}] server.Server(415): Started @122955ms
2024-11-19T00:07:23,949 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:07:24,051 WARN  [Thread-977 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:07:24,055 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa9d6fd1b9d3e5339 with lease ID 0x2cc8ef2c933a09d0: from storage DS-ffef1479-e174-482b-84f8-9ded4c4a4f76 node DatanodeRegistration(127.0.0.1:46809, datanodeUuid=3f4a546e-1f94-4f62-a530-2c54092a62e8, infoPort=38561, infoSecurePort=0, ipcPort=45487, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:07:24,055 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa9d6fd1b9d3e5339 with lease ID 0x2cc8ef2c933a09d0: from storage DS-361069a9-3a24-40e5-9240-1dd6e4cc18f2 node DatanodeRegistration(127.0.0.1:46809, datanodeUuid=3f4a546e-1f94-4f62-a530-2c54092a62e8, infoPort=38561, infoSecurePort=0, ipcPort=45487, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:07:24,449 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@57e35ef0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35029, datanodeUuid=c3327d25-7eb4-4415-97da-c47996dcaff3, infoPort=45405, infoSecurePort=0, ipcPort=43605, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103):Failed to transfer BP-1245155782-172.17.0.2-1731974819103:blk_1073741880_1063 to 127.0.0.1:42365 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:24,449 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@262f9cd[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35029, datanodeUuid=c3327d25-7eb4-4415-97da-c47996dcaff3, infoPort=45405, infoSecurePort=0, ipcPort=43605, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103):Failed to transfer BP-1245155782-172.17.0.2-1731974819103:blk_1073741855_1038 to 127.0.0.1:44795 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:24,943 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:25,725 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:25,928 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:26,449 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@262f9cd[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35029, datanodeUuid=c3327d25-7eb4-4415-97da-c47996dcaff3, infoPort=45405, infoSecurePort=0, ipcPort=43605, storageInfo=lv=-57;cid=testClusterID;nsid=1605392685;c=1731974819103):Failed to transfer BP-1245155782-172.17.0.2-1731974819103:blk_1073741885_1068 to 127.0.0.1:38505 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:26,944 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:27,726 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:27,929 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:28,944 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:29,726 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:29,777 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-19T00:07:29,929 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:30,022 ERROR [FSHLog-0-hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData-prefix:9c732c6543ff,46287,1731974819797 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:30,022 WARN  [FSHLog-0-hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData-prefix:9c732c6543ff,46287,1731974819797 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:30,022 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 9c732c6543ff%2C46287%2C1731974819797:(num 1731974819946) roll requested
2024-11-19T00:07:30,023 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C46287%2C1731974819797.1731974850022
2024-11-19T00:07:30,026 WARN  [Thread-995 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1069
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:30,026 WARN  [Thread-995 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741886_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK], DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]) is bad.
2024-11-19T00:07:30,026 WARN  [Thread-995 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741886_1069
2024-11-19T00:07:30,027 WARN  [Thread-995 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]
2024-11-19T00:07:30,028 WARN  [Thread-995 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1070
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:30,028 WARN  [Thread-995 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741887_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK], DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:30,029 WARN  [Thread-995 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741887_1070
2024-11-19T00:07:30,029 WARN  [Thread-995 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]
2024-11-19T00:07:30,034 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:30,034 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:30,034 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:30,034 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:30,034 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:30,035 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/WALs/9c732c6543ff,46287,1731974819797/9c732c6543ff%2C46287%2C1731974819797.1731974819946 with entries=54, filesize=26.67 KB; new WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/WALs/9c732c6543ff,46287,1731974819797/9c732c6543ff%2C46287%2C1731974819797.1731974850022
2024-11-19T00:07:30,035 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:30,035 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:30,035 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/WALs/9c732c6543ff,46287,1731974819797/9c732c6543ff%2C46287%2C1731974819797.1731974819946
2024-11-19T00:07:30,036 WARN  [IPC Server handler 2 on default port 43835 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/WALs/9c732c6543ff,46287,1731974819797/9c732c6543ff%2C46287%2C1731974819797.1731974819946 has not been closed. Lease recovery is in progress. RecoveryId = 1072 for block blk_1073741830_1006
2024-11-19T00:07:30,036 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/WALs/9c732c6543ff,46287,1731974819797/9c732c6543ff%2C46287%2C1731974819797.1731974819946 after 1ms
2024-11-19T00:07:30,038 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38561:38561),(127.0.0.1/127.0.0.1:45405:45405)]
2024-11-19T00:07:30,038 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/WALs/9c732c6543ff,46287,1731974819797/9c732c6543ff%2C46287%2C1731974819797.1731974819946 is not closed yet, will try archiving it next time
2024-11-19T00:07:30,944 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:31,727 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:32,945 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:33,727 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:34,037 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/WALs/9c732c6543ff,46287,1731974819797/9c732c6543ff%2C46287%2C1731974819797.1731974819946 after 4002ms
2024-11-19T00:07:34,945 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:35,727 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:36,945 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,290 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C42145%2C1731974819848.1731974857290
2024-11-19T00:07:37,293 WARN  [Thread-1004 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741889_1073
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,293 WARN  [Thread-1004 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741889_1073 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK], DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]) is bad.
2024-11-19T00:07:37,293 WARN  [Thread-1004 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741889_1073
2024-11-19T00:07:37,294 WARN  [Thread-1004 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]
2024-11-19T00:07:37,295 WARN  [Thread-1004 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1074
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,296 WARN  [Thread-1004 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741890_1074 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK], DatanodeInfoWithStorage[127.0.0.1:46809,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:37,296 WARN  [Thread-1004 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741890_1074
2024-11-19T00:07:37,296 WARN  [Thread-1004 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]
2024-11-19T00:07:37,302 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:37,303 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:37,303 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:37,303 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:37,303 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:37,303 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974841705 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974857290
2024-11-19T00:07:37,304 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38561:38561),(127.0.0.1/127.0.0.1:45405:45405)]
2024-11-19T00:07:37,304 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245 is not closed yet, will try archiving it next time
2024-11-19T00:07:37,304 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974841705 is not closed yet, will try archiving it next time
2024-11-19T00:07:37,305 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974839687 to hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/oldWALs/9c732c6543ff%2C42145%2C1731974819848.1731974839687
2024-11-19T00:07:37,305 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741875_1058 (size=12911)
2024-11-19T00:07:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42145 {}] regionserver.HRegion(8855): Flush requested on 7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:37,310 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7d4ca27e4c223b1425b85ace7933e5 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-19T00:07:37,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/a72ecf76b4b94165b54b0e8051a40e48 is 1080, key is row0013/info:/1731974857306/Put/seqid=0
2024-11-19T00:07:37,320 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46809 is added to blk_1073741892_1076 (size=8190)
2024-11-19T00:07:37,320 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741892_1076 (size=8190)
2024-11-19T00:07:37,321 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/a72ecf76b4b94165b54b0e8051a40e48
2024-11-19T00:07:37,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/a72ecf76b4b94165b54b0e8051a40e48 as hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/a72ecf76b4b94165b54b0e8051a40e48
2024-11-19T00:07:37,336 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/a72ecf76b4b94165b54b0e8051a40e48, entries=3, sequenceid=66, filesize=8.0 K
2024-11-19T00:07:37,337 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for 7c7d4ca27e4c223b1425b85ace7933e5 in 28ms, sequenceid=66, compaction requested=false
2024-11-19T00:07:37,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7d4ca27e4c223b1425b85ace7933e5:

2024-11-19T00:07:37,338 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K
2024-11-19T00:07:37,338 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:07:37,338 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/4d067a1a9bd44af6920af56aa6df5223 because midkey is the same as first or last row
2024-11-19T00:07:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42145 {}] regionserver.HRegion(8855): Flush requested on 7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:37,534 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7d4ca27e4c223b1425b85ace7933e5 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-19T00:07:37,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/911e63e4a12e492a8dc9869240774ab5 is 1080, key is row0015/info:/1731974857311/Put/seqid=0
2024-11-19T00:07:37,542 WARN  [Thread-1018 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741893_1077
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,542 WARN  [Thread-1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741893_1077 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK], DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]) is bad.
2024-11-19T00:07:37,542 WARN  [Thread-1018 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741893_1077
2024-11-19T00:07:37,542 WARN  [Thread-1018 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]
2024-11-19T00:07:37,544 WARN  [Thread-1018 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1078
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,544 WARN  [Thread-1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741894_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:37,544 WARN  [Thread-1018 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741894_1078
2024-11-19T00:07:37,544 WARN  [Thread-1018 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]
2024-11-19T00:07:37,546 WARN  [Thread-1018 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741895_1079
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,546 WARN  [Thread-1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741895_1079 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK], DatanodeInfoWithStorage[127.0.0.1:46809,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]) is bad.
2024-11-19T00:07:37,546 WARN  [Thread-1018 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741895_1079
2024-11-19T00:07:37,546 WARN  [Thread-1018 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]
2024-11-19T00:07:37,551 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46809 is added to blk_1073741896_1080 (size=14660)
2024-11-19T00:07:37,551 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741896_1080 (size=14660)
2024-11-19T00:07:37,552 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/911e63e4a12e492a8dc9869240774ab5
2024-11-19T00:07:37,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/911e63e4a12e492a8dc9869240774ab5 as hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/911e63e4a12e492a8dc9869240774ab5
2024-11-19T00:07:37,566 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/911e63e4a12e492a8dc9869240774ab5, entries=9, sequenceid=79, filesize=14.3 K
2024-11-19T00:07:37,568 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10758, heapSize ~11.48 KB/11760, currentSize=0 B/0 for 7c7d4ca27e4c223b1425b85ace7933e5 in 33ms, sequenceid=79, compaction requested=true
2024-11-19T00:07:37,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7d4ca27e4c223b1425b85ace7933e5:

2024-11-19T00:07:37,568 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.0 K, sizeToCheck=16.0 K
2024-11-19T00:07:37,568 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:07:37,568 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/4d067a1a9bd44af6920af56aa6df5223 because midkey is the same as first or last row
2024-11-19T00:07:37,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7d4ca27e4c223b1425b85ace7933e5:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T00:07:37,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:07:37,568 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T00:07:37,569 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 40947 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T00:07:37,570 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HStore(1541): 7c7d4ca27e4c223b1425b85ace7933e5/info is initiating minor compaction (all files)
2024-11-19T00:07:37,570 INFO  [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7d4ca27e4c223b1425b85ace7933e5/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.
2024-11-19T00:07:37,570 INFO  [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/4d067a1a9bd44af6920af56aa6df5223, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/a72ecf76b4b94165b54b0e8051a40e48, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/911e63e4a12e492a8dc9869240774ab5] into tmpdir=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp, totalSize=40.0 K
2024-11-19T00:07:37,570 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d067a1a9bd44af6920af56aa6df5223, keycount=12, bloomtype=ROW, size=17.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731974833676
2024-11-19T00:07:37,571 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] compactions.Compactor(225): Compacting a72ecf76b4b94165b54b0e8051a40e48, keycount=3, bloomtype=ROW, size=8.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731974843611
2024-11-19T00:07:37,571 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] compactions.Compactor(225): Compacting 911e63e4a12e492a8dc9869240774ab5, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731974857311
2024-11-19T00:07:37,584 INFO  [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7d4ca27e4c223b1425b85ace7933e5#info#compaction#27 average throughput is 22.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T00:07:37,585 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/a7ab031277ab4eaf88316e62b42b5794 is 1080, key is row0002/info:/1731974833676/Put/seqid=0
2024-11-19T00:07:37,587 WARN  [Thread-1025 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741897_1081
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,587 WARN  [Thread-1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741897_1081 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK], DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]) is bad.
2024-11-19T00:07:37,587 WARN  [Thread-1025 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741897_1081
2024-11-19T00:07:37,587 WARN  [Thread-1025 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]
2024-11-19T00:07:37,588 WARN  [Thread-1025 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741898_1082
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,589 WARN  [Thread-1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741898_1082 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK], DatanodeInfoWithStorage[127.0.0.1:46809,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:37,589 WARN  [Thread-1025 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741898_1082
2024-11-19T00:07:37,589 WARN  [Thread-1025 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]
2024-11-19T00:07:37,590 WARN  [Thread-1025 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741899_1083
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,590 WARN  [Thread-1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741899_1083 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK], DatanodeInfoWithStorage[127.0.0.1:46809,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]) is bad.
2024-11-19T00:07:37,591 WARN  [Thread-1025 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741899_1083
2024-11-19T00:07:37,591 WARN  [Thread-1025 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]
2024-11-19T00:07:37,596 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741900_1084 (size=28989)
2024-11-19T00:07:37,597 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46809 is added to blk_1073741900_1084 (size=28989)
2024-11-19T00:07:37,603 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/.tmp/info/a7ab031277ab4eaf88316e62b42b5794 as hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/a7ab031277ab4eaf88316e62b42b5794
2024-11-19T00:07:37,612 INFO  [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c7d4ca27e4c223b1425b85ace7933e5/info of 7c7d4ca27e4c223b1425b85ace7933e5 into a7ab031277ab4eaf88316e62b42b5794(size=28.3 K), total size for store is 28.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T00:07:37,612 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7d4ca27e4c223b1425b85ace7933e5:

2024-11-19T00:07:37,612 INFO  [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5., storeName=7c7d4ca27e4c223b1425b85ace7933e5/info, priority=13, startTime=1731974857568; duration=0sec
2024-11-19T00:07:37,612 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-19T00:07:37,612 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:07:37,613 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/a7ab031277ab4eaf88316e62b42b5794 because midkey is the same as first or last row
2024-11-19T00:07:37,613 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-19T00:07:37,613 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:07:37,613 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/a7ab031277ab4eaf88316e62b42b5794 because midkey is the same as first or last row
2024-11-19T00:07:37,613 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-19T00:07:37,613 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:07:37,613 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/a7ab031277ab4eaf88316e62b42b5794 because midkey is the same as first or last row
2024-11-19T00:07:37,613 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:07:37,613 DEBUG [RS:0;9c732c6543ff:42145-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7d4ca27e4c223b1425b85ace7933e5:info
2024-11-19T00:07:37,706 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245 is not closed yet, will try archiving it next time
2024-11-19T00:07:37,706 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974841705 to hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/oldWALs/9c732c6543ff%2C42145%2C1731974819848.1731974841705
2024-11-19T00:07:37,728 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-11-19T00:07:37,728 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,734 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-19T00:07:37,735 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T00:07:37,735 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:07:37,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:07:37,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:07:37,735 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-19T00:07:37,736 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-19T00:07:37,736 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1750200013, stopped=false
2024-11-19T00:07:37,736 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9c732c6543ff,46287,1731974819797
2024-11-19T00:07:37,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T00:07:37,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T00:07:37,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:37,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:37,738 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T00:07:37,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42069-0x1010851646e0002, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T00:07:37,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42069-0x1010851646e0002, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:37,738 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T00:07:37,738 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:07:37,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:07:37,739 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:07:37,739 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9c732c6543ff,42145,1731974819848' *****
2024-11-19T00:07:37,739 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-19T00:07:37,739 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9c732c6543ff,42069,1731974820802' *****
2024-11-19T00:07:37,739 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-19T00:07:37,739 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-19T00:07:37,739 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-19T00:07:37,739 INFO  [RS:0;9c732c6543ff:42145 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-19T00:07:37,739 INFO  [RS:0;9c732c6543ff:42145 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-19T00:07:37,739 INFO  [RS:1;9c732c6543ff:42069 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-19T00:07:37,739 INFO  [RS:1;9c732c6543ff:42069 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-19T00:07:37,739 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.HRegionServer(959): stopping server 9c732c6543ff,42069,1731974820802
2024-11-19T00:07:37,739 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(3091): Received CLOSE for 7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:37,739 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T00:07:37,739 INFO  [RS:1;9c732c6543ff:42069 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9c732c6543ff:42069.
2024-11-19T00:07:37,739 DEBUG [RS:1;9c732c6543ff:42069 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:07:37,740 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-19T00:07:37,740 DEBUG [RS:1;9c732c6543ff:42069 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:07:37,740 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-19T00:07:37,740 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.HRegionServer(976): stopping server 9c732c6543ff,42069,1731974820802; all regions closed.
2024-11-19T00:07:37,740 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:07:37,740 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42069-0x1010851646e0002, quorum=127.0.0.1:49660, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:07:37,740 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(959): stopping server 9c732c6543ff,42145,1731974819848
2024-11-19T00:07:37,740 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T00:07:37,740 INFO  [RS:0;9c732c6543ff:42145 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9c732c6543ff:42145.
2024-11-19T00:07:37,740 DEBUG [RS:0;9c732c6543ff:42145 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:07:37,740 DEBUG [RS:0;9c732c6543ff:42145 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:07:37,740 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-19T00:07:37,740 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-19T00:07:37,740 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-19T00:07:37,740 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-19T00:07:37,741 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7c7d4ca27e4c223b1425b85ace7933e5, disabling compactions & flushes
2024-11-19T00:07:37,741 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.
2024-11-19T00:07:37,741 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.
2024-11-19T00:07:37,741 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5. after waiting 0 ms
2024-11-19T00:07:37,741 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.
2024-11-19T00:07:37,743 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:37,743 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-19T00:07:37,743 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:37,743 DEBUG [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(1325): Online Regions={7c7d4ca27e4c223b1425b85ace7933e5=TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5., 1588230740=hbase:meta,,1.1588230740}
2024-11-19T00:07:37,743 DEBUG [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 7c7d4ca27e4c223b1425b85ace7933e5
2024-11-19T00:07:37,743 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:37,743 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T00:07:37,743 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T00:07:37,743 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:37,743 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T00:07:37,743 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T00:07:37,743 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T00:07:37,743 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:37,743 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-11-19T00:07:37,743 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/6ebdd5c2b89546ebbab5c2e4086ac36c, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/96748ab60dd541d38a53ac3eced4fc48, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/e7e854e9361b458d9eee770570edc73c, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/c9f908752b304476a1c03b560f8fb5fb, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/af8c5b31d884402494046d88d18b49d3, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/4d067a1a9bd44af6920af56aa6df5223, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/2ce9a7821392416786ccf9a60bd4b21d, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/a72ecf76b4b94165b54b0e8051a40e48, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/911e63e4a12e492a8dc9869240774ab5] to archive
2024-11-19T00:07:37,744 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,744 ERROR [FSHLog-0-hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35-prefix:9c732c6543ff,42145,1731974819848.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,744 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,744 WARN  [FSHLog-0-hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35-prefix:9c732c6543ff,42145,1731974819848.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,744 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
2024-11-19T00:07:37,744 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9c732c6543ff%2C42145%2C1731974819848.meta:.meta(num 1731974820639) roll requested
2024-11-19T00:07:37,744 INFO  [regionserver/9c732c6543ff:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C42145%2C1731974819848.meta.1731974857744.meta
2024-11-19T00:07:37,745 WARN  [IPC Server handler 1 on default port 43835 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088 has not been closed. Lease recovery is in progress. RecoveryId = 1085 for block blk_1073741837_1013
2024-11-19T00:07:37,745 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088 after 1ms
2024-11-19T00:07:37,745 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-19T00:07:37,749 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/6ebdd5c2b89546ebbab5c2e4086ac36c to hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/6ebdd5c2b89546ebbab5c2e4086ac36c
2024-11-19T00:07:37,750 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/96748ab60dd541d38a53ac3eced4fc48 to hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/96748ab60dd541d38a53ac3eced4fc48
2024-11-19T00:07:37,752 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/e7e854e9361b458d9eee770570edc73c to hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/e7e854e9361b458d9eee770570edc73c
2024-11-19T00:07:37,754 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/c9f908752b304476a1c03b560f8fb5fb to hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/c9f908752b304476a1c03b560f8fb5fb
2024-11-19T00:07:37,755 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/af8c5b31d884402494046d88d18b49d3 to hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/af8c5b31d884402494046d88d18b49d3
2024-11-19T00:07:37,756 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:37,757 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:37,757 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:37,757 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:37,757 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:37,757 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/4d067a1a9bd44af6920af56aa6df5223 to hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/4d067a1a9bd44af6920af56aa6df5223
2024-11-19T00:07:37,757 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974857744.meta
2024-11-19T00:07:37,758 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,758 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,758 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
2024-11-19T00:07:37,759 WARN  [IPC Server handler 1 on default port 43835 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta has not been closed. Lease recovery is in progress. RecoveryId = 1087 for block blk_1073741834_1010
2024-11-19T00:07:37,759 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/2ce9a7821392416786ccf9a60bd4b21d to hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/2ce9a7821392416786ccf9a60bd4b21d
2024-11-19T00:07:37,759 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta after 1ms
2024-11-19T00:07:37,761 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/a72ecf76b4b94165b54b0e8051a40e48 to hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/a72ecf76b4b94165b54b0e8051a40e48
2024-11-19T00:07:37,762 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45405:45405),(127.0.0.1/127.0.0.1:38561:38561)]
2024-11-19T00:07:37,762 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta is not closed yet, will try archiving it next time
2024-11-19T00:07:37,763 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/911e63e4a12e492a8dc9869240774ab5 to hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/info/911e63e4a12e492a8dc9869240774ab5
2024-11-19T00:07:37,763 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=9c732c6543ff:46287 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-19T00:07:37,764 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [6ebdd5c2b89546ebbab5c2e4086ac36c=10347, 96748ab60dd541d38a53ac3eced4fc48=12506, e7e854e9361b458d9eee770570edc73c=17994, c9f908752b304476a1c03b560f8fb5fb=6027, af8c5b31d884402494046d88d18b49d3=6027, 4d067a1a9bd44af6920af56aa6df5223=18097, 2ce9a7821392416786ccf9a60bd4b21d=6027, a72ecf76b4b94165b54b0e8051a40e48=8190, 911e63e4a12e492a8dc9869240774ab5=14660]
2024-11-19T00:07:37,769 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7c7d4ca27e4c223b1425b85ace7933e5/recovered.edits/83.seqid, newMaxSeqId=83, maxSeqId=1
2024-11-19T00:07:37,770 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.
2024-11-19T00:07:37,770 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7c7d4ca27e4c223b1425b85ace7933e5:
Waiting for close lock at 1731974857741Running coprocessor pre-close hooks at 1731974857741Disabling compacts and flushes for region at 1731974857741Disabling writes for close at 1731974857741Writing region close event to WAL at 1731974857764 (+23 ms)Running coprocessor post-close hooks at 1731974857769 (+5 ms)Closed at 1731974857770 (+1 ms)
2024-11-19T00:07:37,770 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5.
2024-11-19T00:07:37,784 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740/.tmp/info/050b98cbb85b49ba8c27bf1ef86fe645 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1731974820988.7c7d4ca27e4c223b1425b85ace7933e5./info:regioninfo/1731974821375/Put/seqid=0
2024-11-19T00:07:37,786 WARN  [Thread-1037 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741902_1088
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,786 WARN  [Thread-1037 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741902_1088 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:37,786 WARN  [Thread-1037 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741902_1088
2024-11-19T00:07:37,786 WARN  [Thread-1037 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]
2024-11-19T00:07:37,789 WARN  [Thread-1037 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741903_1089
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42365
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,789 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:51130 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741903_1089] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data4]'}, localName='127.0.0.1:46809', datanodeUuid='3f4a546e-1f94-4f62-a530-2c54092a62e8', xmitsInProgress=0}:Exception transferring block BP-1245155782-172.17.0.2-1731974819103:blk_1073741903_1089 to mirror 127.0.0.1:42365
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:37,789 WARN  [Thread-1037 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741903_1089 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46809,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK], DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]) is bad.
2024-11-19T00:07:37,789 WARN  [Thread-1037 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741903_1089
2024-11-19T00:07:37,789 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:51130 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741903_1089] {}] datanode.BlockReceiver(316): Block 1073741903 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T00:07:37,790 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:51130 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741903_1089] {}] datanode.DataXceiver(331): 127.0.0.1:46809:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51130 dst: /127.0.0.1:46809
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:37,790 WARN  [Thread-1037 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]
2024-11-19T00:07:37,791 WARN  [Thread-1037 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741904_1090
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,791 WARN  [Thread-1037 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741904_1090 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK], DatanodeInfoWithStorage[127.0.0.1:46809,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]) is bad.
2024-11-19T00:07:37,791 WARN  [Thread-1037 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741904_1090
2024-11-19T00:07:37,792 WARN  [Thread-1037 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]
2024-11-19T00:07:37,797 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46809 is added to blk_1073741905_1091 (size=7089)
2024-11-19T00:07:37,797 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741905_1091 (size=7089)
2024-11-19T00:07:37,797 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740/.tmp/info/050b98cbb85b49ba8c27bf1ef86fe645
2024-11-19T00:07:37,820 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740/.tmp/ns/8bc5911cea7f49ec8df47529c3830021 is 43, key is default/ns:d/1731974820720/Put/seqid=0
2024-11-19T00:07:37,823 WARN  [Thread-1045 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741906_1092
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42365
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,823 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:60134 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741906_1092] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data8]'}, localName='127.0.0.1:35029', datanodeUuid='c3327d25-7eb4-4415-97da-c47996dcaff3', xmitsInProgress=0}:Exception transferring block BP-1245155782-172.17.0.2-1731974819103:blk_1073741906_1092 to mirror 127.0.0.1:42365
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:37,824 WARN  [Thread-1045 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741906_1092 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK], DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]) is bad.
2024-11-19T00:07:37,824 WARN  [Thread-1045 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741906_1092
2024-11-19T00:07:37,824 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:60134 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741906_1092] {}] datanode.BlockReceiver(316): Block 1073741906 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T00:07:37,824 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:60134 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741906_1092] {}] datanode.DataXceiver(331): 127.0.0.1:35029:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60134 dst: /127.0.0.1:35029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:37,825 WARN  [Thread-1045 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]
2024-11-19T00:07:37,826 WARN  [Thread-1045 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741907_1093
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,826 WARN  [Thread-1045 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741907_1093 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK], DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:37,826 WARN  [Thread-1045 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741907_1093
2024-11-19T00:07:37,826 WARN  [Thread-1045 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]
2024-11-19T00:07:37,827 WARN  [Thread-1045 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741908_1094
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:37,827 WARN  [Thread-1045 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741908_1094 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK], DatanodeInfoWithStorage[127.0.0.1:46809,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]) is bad.
2024-11-19T00:07:37,828 WARN  [Thread-1045 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741908_1094
2024-11-19T00:07:37,828 WARN  [Thread-1045 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]
2024-11-19T00:07:37,835 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741909_1095 (size=5153)
2024-11-19T00:07:37,835 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46809 is added to blk_1073741909_1095 (size=5153)
2024-11-19T00:07:37,943 DEBUG [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-19T00:07:37,989 INFO  [regionserver/9c732c6543ff:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-19T00:07:37,989 INFO  [regionserver/9c732c6543ff:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-19T00:07:38,105 INFO  [regionserver/9c732c6543ff:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-19T00:07:38,106 INFO  [regionserver/9c732c6543ff:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-19T00:07:38,107 INFO  [regionserver/9c732c6543ff:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T00:07:38,143 DEBUG [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-19T00:07:38,235 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740/.tmp/ns/8bc5911cea7f49ec8df47529c3830021
2024-11-19T00:07:38,258 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740/.tmp/table/988ed49a888a46188fcc0dab2e2c4aa8 is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1731974821395/Put/seqid=0
2024-11-19T00:07:38,261 WARN  [Thread-1053 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741910_1096
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:38,261 WARN  [Thread-1053 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741910_1096 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK], DatanodeInfoWithStorage[127.0.0.1:46809,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]) is bad.
2024-11-19T00:07:38,261 WARN  [Thread-1053 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741910_1096
2024-11-19T00:07:38,262 WARN  [Thread-1053 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42365,DS-f4aa5c1a-6aa7-43a9-aa26-eb2cbfe8efae,DISK]
2024-11-19T00:07:38,264 WARN  [Thread-1053 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741911_1097
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38505
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:38,264 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:51148 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741911_1097] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data4]'}, localName='127.0.0.1:46809', datanodeUuid='3f4a546e-1f94-4f62-a530-2c54092a62e8', xmitsInProgress=0}:Exception transferring block BP-1245155782-172.17.0.2-1731974819103:blk_1073741911_1097 to mirror 127.0.0.1:38505
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:38,264 WARN  [Thread-1053 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741911_1097 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46809,DS-ffef1479-e174-482b-84f8-9ded4c4a4f76,DISK], DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]) is bad.
2024-11-19T00:07:38,264 WARN  [Thread-1053 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741911_1097
2024-11-19T00:07:38,264 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:51148 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741911_1097] {}] datanode.BlockReceiver(316): Block 1073741911 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T00:07:38,264 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1632792343_22 at /127.0.0.1:51148 [Receiving block BP-1245155782-172.17.0.2-1731974819103:blk_1073741911_1097] {}] datanode.DataXceiver(331): 127.0.0.1:46809:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51148 dst: /127.0.0.1:46809
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:38,265 WARN  [Thread-1053 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38505,DS-f7a5953f-ad89-43e6-a049-90082b0ad605,DISK]
2024-11-19T00:07:38,266 WARN  [Thread-1053 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741912_1098
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:07:38,266 WARN  [Thread-1053 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1245155782-172.17.0.2-1731974819103:blk_1073741912_1098 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK], DatanodeInfoWithStorage[127.0.0.1:35029,DS-6ca4b7d9-870f-4ac6-b9f9-2ac210d02cb4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]) is bad.
2024-11-19T00:07:38,266 WARN  [Thread-1053 {}] hdfs.DataStreamer(1850): Abandoning BP-1245155782-172.17.0.2-1731974819103:blk_1073741912_1098
2024-11-19T00:07:38,266 WARN  [Thread-1053 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44795,DS-b692e405-1e4d-4f68-9d27-eaee00fbe866,DISK]
2024-11-19T00:07:38,271 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46809 is added to blk_1073741913_1099 (size=5424)
2024-11-19T00:07:38,271 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741913_1099 (size=5424)
2024-11-19T00:07:38,271 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740/.tmp/table/988ed49a888a46188fcc0dab2e2c4aa8
2024-11-19T00:07:38,278 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740/.tmp/info/050b98cbb85b49ba8c27bf1ef86fe645 as hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740/info/050b98cbb85b49ba8c27bf1ef86fe645
2024-11-19T00:07:38,284 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740/info/050b98cbb85b49ba8c27bf1ef86fe645, entries=10, sequenceid=11, filesize=6.9 K
2024-11-19T00:07:38,285 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740/.tmp/ns/8bc5911cea7f49ec8df47529c3830021 as hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740/ns/8bc5911cea7f49ec8df47529c3830021
2024-11-19T00:07:38,291 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740/ns/8bc5911cea7f49ec8df47529c3830021, entries=2, sequenceid=11, filesize=5.0 K
2024-11-19T00:07:38,292 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740/.tmp/table/988ed49a888a46188fcc0dab2e2c4aa8 as hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740/table/988ed49a888a46188fcc0dab2e2c4aa8
2024-11-19T00:07:38,298 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740/table/988ed49a888a46188fcc0dab2e2c4aa8, entries=2, sequenceid=11, filesize=5.3 K
2024-11-19T00:07:38,299 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 556ms, sequenceid=11, compaction requested=false
2024-11-19T00:07:38,304 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-19T00:07:38,305 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T00:07:38,305 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T00:07:38,305 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731974857743Running coprocessor pre-close hooks at 1731974857743Disabling compacts and flushes for region at 1731974857743Disabling writes for close at 1731974857743Obtaining lock to block concurrent updates at 1731974857743Preparing flush snapshotting stores in 1588230740 at 1731974857743Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1731974857744 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731974857762 (+18 ms)Flushing 1588230740/info: creating writer at 1731974857762Flushing 1588230740/info: appending metadata at 1731974857783 (+21 ms)Flushing 1588230740/info: closing flushed file at 1731974857784 (+1 ms)Flushing 1588230740/ns: creating writer at 1731974857804 (+20 ms)Flushing 1588230740/ns: appending metadata at 1731974857820 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731974857820Flushing 1588230740/table: creating writer at 1731974858242 (+422 ms)Flushing 1588230740/table: appending metadata at 1731974858257 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731974858257Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@156421d4: reopening flushed file at 1731974858276 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b6199f5: reopening flushed file at 1731974858284 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50a84337: reopening flushed file at 1731974858291 (+7 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 556ms, sequenceid=11, compaction requested=false at 1731974858299 (+8 ms)Writing region close event to WAL at 1731974858301 (+2 ms)Running coprocessor post-close hooks at 1731974858305 (+4 ms)Closed at 1731974858305
2024-11-19T00:07:38,305 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-19T00:07:38,344 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(976): stopping server 9c732c6543ff,42145,1731974819848; all regions closed.
2024-11-19T00:07:38,344 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:38,344 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:38,344 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:38,345 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:38,345 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:38,347 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46809 is added to blk_1073741901_1086 (size=825)
2024-11-19T00:07:38,347 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741901_1086 (size=825)
2024-11-19T00:07:38,454 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46809 is added to blk_1073741875_1058 (size=12911)
2024-11-19T00:07:38,943 INFO  [regionserver/9c732c6543ff:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T00:07:40,077 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@7e61afec {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1245155782-172.17.0.2-1731974819103:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:42365,null,null])
java.net.ConnectException: Call From 9c732c6543ff/172.17.0.2 to localhost:41299 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-19T00:07:40,079 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46809 is added to blk_1073741830_1072 (size=27306)
2024-11-19T00:07:40,741 INFO  [master/9c732c6543ff:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-19T00:07:40,741 INFO  [master/9c732c6543ff:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-19T00:07:41,746 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088 after 4002ms
2024-11-19T00:07:41,760 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta after 4002ms
2024-11-19T00:07:42,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741831_1007 (size=1321)
2024-11-19T00:07:42,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741835_1011 (size=393)
2024-11-19T00:07:42,744 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-19T00:07:42,746 DEBUG [RS:1;9c732c6543ff:42069 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/oldWALs
2024-11-19T00:07:42,746 INFO  [RS:1;9c732c6543ff:42069 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9c732c6543ff%2C42069%2C1731974820802:(num 1731974821088)
2024-11-19T00:07:42,746 DEBUG [RS:1;9c732c6543ff:42069 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:07:42,746 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T00:07:42,747 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T00:07:42,747 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.ChoreService(370): Chore service for: regionserver/9c732c6543ff:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-19T00:07:42,747 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-19T00:07:42,747 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-19T00:07:42,747 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-19T00:07:42,747 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T00:07:42,747 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T00:07:42,747 INFO  [RS:1;9c732c6543ff:42069 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42069
2024-11-19T00:07:42,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42069-0x1010851646e0002, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9c732c6543ff,42069,1731974820802
2024-11-19T00:07:42,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T00:07:42,750 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T00:07:42,752 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9c732c6543ff,42069,1731974820802]
2024-11-19T00:07:42,752 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:42,754 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9c732c6543ff,42069,1731974820802 already deleted, retry=false
2024-11-19T00:07:42,754 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9c732c6543ff,42069,1731974820802 expired; onlineServers=1
2024-11-19T00:07:42,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:42,787 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:42,788 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:42,788 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:42,788 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:42,789 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:42,795 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:42,797 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:42,852 INFO  [RS:1;9c732c6543ff:42069 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T00:07:42,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42069-0x1010851646e0002, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:07:42,852 INFO  [RS:1;9c732c6543ff:42069 {}] regionserver.HRegionServer(1031): Exiting; stopping=9c732c6543ff,42069,1731974820802; zookeeper connection closed.
2024-11-19T00:07:42,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42069-0x1010851646e0002, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:07:42,853 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4c1cf3a3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4c1cf3a3
2024-11-19T00:07:43,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741827_1003 (size=196)
2024-11-19T00:07:43,058 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741829_1005 (size=34)
2024-11-19T00:07:43,298 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T00:07:43,314 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:43,315 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:43,315 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:43,315 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:43,316 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:43,316 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:43,319 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:43,321 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:43,345 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-19T00:07:43,350 DEBUG [RS:0;9c732c6543ff:42145 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/oldWALs
2024-11-19T00:07:43,350 INFO  [RS:0;9c732c6543ff:42145 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9c732c6543ff%2C42145%2C1731974819848.meta:.meta(num 1731974857744)
2024-11-19T00:07:43,350 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:43,350 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:43,350 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:43,351 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:43,351 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:43,352 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741891_1075 (size=16308)
2024-11-19T00:07:43,353 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46809 is added to blk_1073741891_1075 (size=16308)
2024-11-19T00:07:43,753 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:43,762 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-19T00:07:43,762 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:44,753 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:44,764 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:45,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741825_1001 (size=7)
2024-11-19T00:07:45,058 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741836_1012 (size=76)
2024-11-19T00:07:45,754 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:45,762 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-19T00:07:45,765 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:46,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741828_1004 (size=1189)
2024-11-19T00:07:46,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741832_1008 (size=32)
2024-11-19T00:07:46,253 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-19T00:07:46,253 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T00:07:46,253 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-19T00:07:46,755 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:46,767 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:47,755 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:47,763 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-19T00:07:47,769 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:48,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741826_1002 (size=42)
2024-11-19T00:07:48,058 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741830_1072 (size=27306)
2024-11-19T00:07:48,351 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-19T00:07:48,355 DEBUG [RS:0;9c732c6543ff:42145 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/oldWALs
2024-11-19T00:07:48,355 INFO  [RS:0;9c732c6543ff:42145 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9c732c6543ff%2C42145%2C1731974819848:(num 1731974857290)
2024-11-19T00:07:48,355 DEBUG [RS:0;9c732c6543ff:42145 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:07:48,355 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T00:07:48,355 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T00:07:48,355 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.ChoreService(370): Chore service for: regionserver/9c732c6543ff:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-19T00:07:48,356 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T00:07:48,356 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T00:07:48,356 INFO  [RS:0;9c732c6543ff:42145 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42145
2024-11-19T00:07:48,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9c732c6543ff,42145,1731974819848
2024-11-19T00:07:48,358 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T00:07:48,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T00:07:48,360 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9c732c6543ff,42145,1731974819848]
2024-11-19T00:07:48,362 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9c732c6543ff,42145,1731974819848 already deleted, retry=false
2024-11-19T00:07:48,362 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9c732c6543ff,42145,1731974819848 expired; onlineServers=0
2024-11-19T00:07:48,362 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9c732c6543ff,46287,1731974819797' *****
2024-11-19T00:07:48,362 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-19T00:07:48,362 INFO  [M:0;9c732c6543ff:46287 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T00:07:48,362 INFO  [M:0;9c732c6543ff:46287 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T00:07:48,362 DEBUG [M:0;9c732c6543ff:46287 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-19T00:07:48,362 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-19T00:07:48,362 DEBUG [M:0;9c732c6543ff:46287 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-19T00:07:48,362 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974820021 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974820021,5,FailOnTimeoutGroup]
2024-11-19T00:07:48,362 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974820021 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974820021,5,FailOnTimeoutGroup]
2024-11-19T00:07:48,363 INFO  [M:0;9c732c6543ff:46287 {}] hbase.ChoreService(370): Chore service for: master/9c732c6543ff:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-19T00:07:48,363 INFO  [M:0;9c732c6543ff:46287 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T00:07:48,363 DEBUG [M:0;9c732c6543ff:46287 {}] master.HMaster(1795): Stopping service threads
2024-11-19T00:07:48,363 INFO  [M:0;9c732c6543ff:46287 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-19T00:07:48,363 INFO  [M:0;9c732c6543ff:46287 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T00:07:48,363 INFO  [M:0;9c732c6543ff:46287 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-19T00:07:48,363 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-19T00:07:48,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-19T00:07:48,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:48,365 DEBUG [M:0;9c732c6543ff:46287 {}] zookeeper.ZKUtil(347): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-19T00:07:48,365 WARN  [M:0;9c732c6543ff:46287 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-19T00:07:48,366 INFO  [M:0;9c732c6543ff:46287 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/.lastflushedseqids
2024-11-19T00:07:48,373 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46809 is added to blk_1073741914_1100 (size=130)
2024-11-19T00:07:48,373 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741914_1100 (size=130)
2024-11-19T00:07:48,374 INFO  [M:0;9c732c6543ff:46287 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-19T00:07:48,374 INFO  [M:0;9c732c6543ff:46287 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-19T00:07:48,374 DEBUG [M:0;9c732c6543ff:46287 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T00:07:48,374 INFO  [M:0;9c732c6543ff:46287 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:07:48,374 DEBUG [M:0;9c732c6543ff:46287 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:07:48,374 DEBUG [M:0;9c732c6543ff:46287 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T00:07:48,374 DEBUG [M:0;9c732c6543ff:46287 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:07:48,375 INFO  [M:0;9c732c6543ff:46287 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.25 KB heapSize=29.49 KB
2024-11-19T00:07:48,393 DEBUG [M:0;9c732c6543ff:46287 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f63c9a1a06dc4fc198e8c4bc71876c46 is 82, key is hbase:meta,,1/info:regioninfo/1731974820698/Put/seqid=0
2024-11-19T00:07:48,403 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741915_1101 (size=5672)
2024-11-19T00:07:48,403 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46809 is added to blk_1073741915_1101 (size=5672)
2024-11-19T00:07:48,404 INFO  [M:0;9c732c6543ff:46287 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f63c9a1a06dc4fc198e8c4bc71876c46
2024-11-19T00:07:48,434 DEBUG [M:0;9c732c6543ff:46287 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/549144fc40d249a09b79ffddcb64c36d is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731974821401/Put/seqid=0
2024-11-19T00:07:48,439 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46809 is added to blk_1073741916_1102 (size=6255)
2024-11-19T00:07:48,440 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741916_1102 (size=6255)
2024-11-19T00:07:48,441 INFO  [M:0;9c732c6543ff:46287 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/549144fc40d249a09b79ffddcb64c36d
2024-11-19T00:07:48,448 INFO  [M:0;9c732c6543ff:46287 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 549144fc40d249a09b79ffddcb64c36d
2024-11-19T00:07:48,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:07:48,460 INFO  [RS:0;9c732c6543ff:42145 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T00:07:48,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42145-0x1010851646e0001, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:07:48,460 INFO  [RS:0;9c732c6543ff:42145 {}] regionserver.HRegionServer(1031): Exiting; stopping=9c732c6543ff,42145,1731974819848; zookeeper connection closed.
2024-11-19T00:07:48,461 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@f9fb3ff {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@f9fb3ff
2024-11-19T00:07:48,461 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-11-19T00:07:48,464 DEBUG [M:0;9c732c6543ff:46287 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fd288bb645bb438cb8935df6a196bfa7 is 69, key is 9c732c6543ff,42069,1731974820802/rs:state/1731974820863/Put/seqid=0
2024-11-19T00:07:48,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741917_1103 (size=5224)
2024-11-19T00:07:48,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46809 is added to blk_1073741917_1103 (size=5224)
2024-11-19T00:07:48,470 INFO  [M:0;9c732c6543ff:46287 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fd288bb645bb438cb8935df6a196bfa7
2024-11-19T00:07:48,490 DEBUG [M:0;9c732c6543ff:46287 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7c74c435ca074c95b2cb077632ecb652 is 52, key is load_balancer_on/state:d/1731974820784/Put/seqid=0
2024-11-19T00:07:48,495 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741918_1104 (size=5056)
2024-11-19T00:07:48,496 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46809 is added to blk_1073741918_1104 (size=5056)
2024-11-19T00:07:48,496 INFO  [M:0;9c732c6543ff:46287 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7c74c435ca074c95b2cb077632ecb652
2024-11-19T00:07:48,502 DEBUG [M:0;9c732c6543ff:46287 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f63c9a1a06dc4fc198e8c4bc71876c46 as hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f63c9a1a06dc4fc198e8c4bc71876c46
2024-11-19T00:07:48,508 INFO  [M:0;9c732c6543ff:46287 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f63c9a1a06dc4fc198e8c4bc71876c46, entries=8, sequenceid=60, filesize=5.5 K
2024-11-19T00:07:48,509 DEBUG [M:0;9c732c6543ff:46287 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/549144fc40d249a09b79ffddcb64c36d as hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/549144fc40d249a09b79ffddcb64c36d
2024-11-19T00:07:48,518 INFO  [M:0;9c732c6543ff:46287 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 549144fc40d249a09b79ffddcb64c36d
2024-11-19T00:07:48,518 INFO  [M:0;9c732c6543ff:46287 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/549144fc40d249a09b79ffddcb64c36d, entries=6, sequenceid=60, filesize=6.1 K
2024-11-19T00:07:48,519 DEBUG [M:0;9c732c6543ff:46287 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fd288bb645bb438cb8935df6a196bfa7 as hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fd288bb645bb438cb8935df6a196bfa7
2024-11-19T00:07:48,524 INFO  [M:0;9c732c6543ff:46287 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fd288bb645bb438cb8935df6a196bfa7, entries=2, sequenceid=60, filesize=5.1 K
2024-11-19T00:07:48,525 DEBUG [M:0;9c732c6543ff:46287 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7c74c435ca074c95b2cb077632ecb652 as hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/7c74c435ca074c95b2cb077632ecb652
2024-11-19T00:07:48,530 INFO  [M:0;9c732c6543ff:46287 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/7c74c435ca074c95b2cb077632ecb652, entries=1, sequenceid=60, filesize=4.9 K
2024-11-19T00:07:48,531 INFO  [M:0;9c732c6543ff:46287 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 157ms, sequenceid=60, compaction requested=false
2024-11-19T00:07:48,537 INFO  [M:0;9c732c6543ff:46287 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:07:48,538 DEBUG [M:0;9c732c6543ff:46287 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731974868374Disabling compacts and flushes for region at 1731974868374Disabling writes for close at 1731974868374Obtaining lock to block concurrent updates at 1731974868375 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731974868375Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23805, getHeapSize=30136, getOffHeapSize=0, getCellsCount=71 at 1731974868375Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731974868376 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731974868376Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731974868393 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731974868393Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731974868411 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731974868433 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731974868433Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731974868448 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731974868463 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731974868463Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731974868475 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731974868490 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731974868490Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a4b1b76: reopening flushed file at 1731974868501 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f82622b: reopening flushed file at 1731974868508 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e553219: reopening flushed file at 1731974868518 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e75b030: reopening flushed file at 1731974868525 (+7 ms)Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 157ms, sequenceid=60, compaction requested=false at 1731974868531 (+6 ms)Writing region close event to WAL at 1731974868537 (+6 ms)Closed at 1731974868537
2024-11-19T00:07:48,539 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:48,539 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:48,539 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:48,539 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:48,540 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:07:48,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35029 is added to blk_1073741888_1071 (size=1045)
2024-11-19T00:07:48,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46809 is added to blk_1073741888_1071 (size=1045)
2024-11-19T00:07:48,542 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T00:07:48,542 INFO  [M:0;9c732c6543ff:46287 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-19T00:07:48,542 INFO  [M:0;9c732c6543ff:46287 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46287
2024-11-19T00:07:48,543 INFO  [M:0;9c732c6543ff:46287 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T00:07:48,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:07:48,645 INFO  [M:0;9c732c6543ff:46287 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T00:07:48,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46287-0x1010851646e0000, quorum=127.0.0.1:49660, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:07:48,647 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@69aa82e5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:07:48,648 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6fc4b42f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:07:48,648 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:07:48,648 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69a3dde8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:07:48,648 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1191c470{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.log.dir/,STOPPED}
2024-11-19T00:07:48,649 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:07:48,649 WARN  [BP-1245155782-172.17.0.2-1731974819103 heartbeating to localhost/127.0.0.1:43835 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:07:48,649 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:07:48,649 WARN  [BP-1245155782-172.17.0.2-1731974819103 heartbeating to localhost/127.0.0.1:43835 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1245155782-172.17.0.2-1731974819103 (Datanode Uuid 3f4a546e-1f94-4f62-a530-2c54092a62e8) service to localhost/127.0.0.1:43835
2024-11-19T00:07:48,649 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@15fd3a49 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1245155782-172.17.0.2-1731974819103:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:42365,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:41299 , LocalHost:localPort 9c732c6543ff/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-19T00:07:48,650 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@15fd3a49 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1245155782-172.17.0.2-1731974819103:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:46809,null,null], DatanodeInfoWithStorage[127.0.0.1:42365,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: No block pool offer service for bpid=BP-1245155782-172.17.0.2-1731974819103
2024-11-19T00:07:48,650 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@15fd3a49 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1245155782-172.17.0.2-1731974819103:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:42365,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1245155782-172.17.0.2-1731974819103
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:48,650 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data3/current/BP-1245155782-172.17.0.2-1731974819103 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:07:48,650 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@15fd3a49 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1245155782-172.17.0.2-1731974819103:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:46809,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1245155782-172.17.0.2-1731974819103
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:07:48,650 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@15fd3a49 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1245155782-172.17.0.2-1731974819103:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:42365,null,null], DatanodeInfoWithStorage[127.0.0.1:46809,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1245155782-172.17.0.2-1731974819103:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:42365,null,null], DatanodeInfoWithStorage[127.0.0.1:46809,null,null]]
2024-11-19T00:07:48,651 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data4/current/BP-1245155782-172.17.0.2-1731974819103 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:07:48,651 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:07:48,653 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@458e5bcf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:07:48,653 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6b75ba45{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:07:48,653 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:07:48,653 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3986ff43{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:07:48,654 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fbc343d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.log.dir/,STOPPED}
2024-11-19T00:07:48,655 WARN  [BP-1245155782-172.17.0.2-1731974819103 heartbeating to localhost/127.0.0.1:43835 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:07:48,655 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:07:48,655 WARN  [BP-1245155782-172.17.0.2-1731974819103 heartbeating to localhost/127.0.0.1:43835 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1245155782-172.17.0.2-1731974819103 (Datanode Uuid c3327d25-7eb4-4415-97da-c47996dcaff3) service to localhost/127.0.0.1:43835
2024-11-19T00:07:48,655 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:07:48,656 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data7/current/BP-1245155782-172.17.0.2-1731974819103 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:07:48,656 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/cluster_eaf78dba-14e4-46f6-d899-82a6832f9c4d/data/data8/current/BP-1245155782-172.17.0.2-1731974819103 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:07:48,656 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:07:48,662 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@56ff2226{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T00:07:48,663 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4b5c1b0d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:07:48,663 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:07:48,663 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5d8a9c69{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:07:48,663 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@35c95cb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.log.dir/,STOPPED}
2024-11-19T00:07:48,672 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-19T00:07:48,700 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:48,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-19T00:07:48,712 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=155 (was 79)
Potentially hanging thread: LeaseRenewer:jenkins@localhost:43835
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43835
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f0348befdc8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43835 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43835 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:43961
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:43835
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:43961
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43835
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:43835
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43835 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43835
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f0348befdc8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43835
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f0348befdc8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43835 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-16-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=436 (was 402) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 99) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4865 (was 4547) - AvailableMemoryMB LEAK? -
2024-11-19T00:07:48,719 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=155, OpenFileDescriptor=436, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=4865
2024-11-19T00:07:48,720 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-19T00:07:48,720 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.log.dir so I do NOT create it in target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f
2024-11-19T00:07:48,720 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9989f7e3-3a07-44e9-d00e-5bfe619d2848/hadoop.tmp.dir so I do NOT create it in target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f
2024-11-19T00:07:48,720 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e, deleteOnExit=true
2024-11-19T00:07:48,720 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-19T00:07:48,720 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/test.cache.data in system properties and HBase conf
2024-11-19T00:07:48,720 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.tmp.dir in system properties and HBase conf
2024-11-19T00:07:48,720 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.log.dir in system properties and HBase conf
2024-11-19T00:07:48,720 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-19T00:07:48,720 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-19T00:07:48,720 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-19T00:07:48,720 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-19T00:07:48,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-19T00:07:48,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-19T00:07:48,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-19T00:07:48,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T00:07:48,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-19T00:07:48,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-19T00:07:48,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T00:07:48,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T00:07:48,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-19T00:07:48,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/nfs.dump.dir in system properties and HBase conf
2024-11-19T00:07:48,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/java.io.tmpdir in system properties and HBase conf
2024-11-19T00:07:48,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T00:07:48,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-19T00:07:48,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-19T00:07:48,736 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T00:07:48,756 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:48,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:48,811 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:07:48,815 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:07:48,816 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:07:48,816 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:07:48,816 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T00:07:48,817 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:07:48,819 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1915705e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:07:48,819 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@347a2271{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:07:48,956 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@16304a50{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/java.io.tmpdir/jetty-localhost-43403-hadoop-hdfs-3_4_1-tests_jar-_-any-6161611825970398818/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T00:07:48,957 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e4fac25{HTTP/1.1, (http/1.1)}{localhost:43403}
2024-11-19T00:07:48,957 INFO  [Time-limited test {}] server.Server(415): Started @147964ms
2024-11-19T00:07:48,973 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T00:07:49,055 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:07:49,058 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:07:49,059 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:07:49,059 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:07:49,059 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T00:07:49,059 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cc14744{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:07:49,060 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fefca8b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:07:49,177 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@21865735{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/java.io.tmpdir/jetty-localhost-37053-hadoop-hdfs-3_4_1-tests_jar-_-any-14478663280352347176/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:07:49,178 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@54562211{HTTP/1.1, (http/1.1)}{localhost:37053}
2024-11-19T00:07:49,178 INFO  [Time-limited test {}] server.Server(415): Started @148185ms
2024-11-19T00:07:49,180 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:07:49,218 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:07:49,221 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:07:49,222 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:07:49,222 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:07:49,222 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T00:07:49,223 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38cf0a15{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:07:49,223 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a9fc515{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:07:49,272 WARN  [Thread-1183 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/data/data1/current/BP-1256842488-172.17.0.2-1731974868755/current, will proceed with Du for space computation calculation, 
2024-11-19T00:07:49,273 WARN  [Thread-1184 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/data/data2/current/BP-1256842488-172.17.0.2-1731974868755/current, will proceed with Du for space computation calculation, 
2024-11-19T00:07:49,299 WARN  [Thread-1162 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:07:49,301 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x369e8f9e60150ddb with lease ID 0x773b84388bc8ab53: Processing first storage report for DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac from datanode DatanodeRegistration(127.0.0.1:32833, datanodeUuid=ea9b0a6a-d45f-4c45-aa69-040e1f5651cd, infoPort=37281, infoSecurePort=0, ipcPort=38629, storageInfo=lv=-57;cid=testClusterID;nsid=615369472;c=1731974868755)
2024-11-19T00:07:49,302 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x369e8f9e60150ddb with lease ID 0x773b84388bc8ab53: from storage DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac node DatanodeRegistration(127.0.0.1:32833, datanodeUuid=ea9b0a6a-d45f-4c45-aa69-040e1f5651cd, infoPort=37281, infoSecurePort=0, ipcPort=38629, storageInfo=lv=-57;cid=testClusterID;nsid=615369472;c=1731974868755), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:07:49,302 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x369e8f9e60150ddb with lease ID 0x773b84388bc8ab53: Processing first storage report for DS-5e6cb786-f3a8-4e4b-ab57-eb625a535996 from datanode DatanodeRegistration(127.0.0.1:32833, datanodeUuid=ea9b0a6a-d45f-4c45-aa69-040e1f5651cd, infoPort=37281, infoSecurePort=0, ipcPort=38629, storageInfo=lv=-57;cid=testClusterID;nsid=615369472;c=1731974868755)
2024-11-19T00:07:49,302 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x369e8f9e60150ddb with lease ID 0x773b84388bc8ab53: from storage DS-5e6cb786-f3a8-4e4b-ab57-eb625a535996 node DatanodeRegistration(127.0.0.1:32833, datanodeUuid=ea9b0a6a-d45f-4c45-aa69-040e1f5651cd, infoPort=37281, infoSecurePort=0, ipcPort=38629, storageInfo=lv=-57;cid=testClusterID;nsid=615369472;c=1731974868755), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:07:49,367 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1fc8bed8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/java.io.tmpdir/jetty-localhost-46725-hadoop-hdfs-3_4_1-tests_jar-_-any-10800827673295319642/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:07:49,368 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@44bd8b73{HTTP/1.1, (http/1.1)}{localhost:46725}
2024-11-19T00:07:49,368 INFO  [Time-limited test {}] server.Server(415): Started @148375ms
2024-11-19T00:07:49,370 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:07:49,471 WARN  [Thread-1209 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/data/data3/current/BP-1256842488-172.17.0.2-1731974868755/current, will proceed with Du for space computation calculation, 
2024-11-19T00:07:49,472 WARN  [Thread-1210 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/data/data4/current/BP-1256842488-172.17.0.2-1731974868755/current, will proceed with Du for space computation calculation, 
2024-11-19T00:07:49,494 WARN  [Thread-1198 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:07:49,497 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7d76d4df9bd8f836 with lease ID 0x773b84388bc8ab54: Processing first storage report for DS-26ae9a1c-e803-469a-8dfe-b9cfac092529 from datanode DatanodeRegistration(127.0.0.1:38767, datanodeUuid=03fef0cd-5312-4870-ab41-4af83df32bee, infoPort=44891, infoSecurePort=0, ipcPort=43879, storageInfo=lv=-57;cid=testClusterID;nsid=615369472;c=1731974868755)
2024-11-19T00:07:49,497 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7d76d4df9bd8f836 with lease ID 0x773b84388bc8ab54: from storage DS-26ae9a1c-e803-469a-8dfe-b9cfac092529 node DatanodeRegistration(127.0.0.1:38767, datanodeUuid=03fef0cd-5312-4870-ab41-4af83df32bee, infoPort=44891, infoSecurePort=0, ipcPort=43879, storageInfo=lv=-57;cid=testClusterID;nsid=615369472;c=1731974868755), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:07:49,497 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7d76d4df9bd8f836 with lease ID 0x773b84388bc8ab54: Processing first storage report for DS-9cc89262-61b5-46ed-8c03-ec3ce055a0f2 from datanode DatanodeRegistration(127.0.0.1:38767, datanodeUuid=03fef0cd-5312-4870-ab41-4af83df32bee, infoPort=44891, infoSecurePort=0, ipcPort=43879, storageInfo=lv=-57;cid=testClusterID;nsid=615369472;c=1731974868755)
2024-11-19T00:07:49,497 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7d76d4df9bd8f836 with lease ID 0x773b84388bc8ab54: from storage DS-9cc89262-61b5-46ed-8c03-ec3ce055a0f2 node DatanodeRegistration(127.0.0.1:38767, datanodeUuid=03fef0cd-5312-4870-ab41-4af83df32bee, infoPort=44891, infoSecurePort=0, ipcPort=43879, storageInfo=lv=-57;cid=testClusterID;nsid=615369472;c=1731974868755), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:07:49,500 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f
2024-11-19T00:07:49,505 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/zookeeper_0, clientPort=52353, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-19T00:07:49,506 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52353
2024-11-19T00:07:49,506 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:07:49,508 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:07:49,520 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32833 is added to blk_1073741825_1001 (size=7)
2024-11-19T00:07:49,521 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38767 is added to blk_1073741825_1001 (size=7)
2024-11-19T00:07:49,522 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9 with version=8
2024-11-19T00:07:49,522 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/hbase-staging
2024-11-19T00:07:49,524 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/9c732c6543ff:0 server-side Connection retries=45
2024-11-19T00:07:49,525 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:07:49,525 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T00:07:49,525 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T00:07:49,525 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:07:49,525 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T00:07:49,525 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-19T00:07:49,525 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T00:07:49,526 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33509
2024-11-19T00:07:49,527 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33509 connecting to ZooKeeper ensemble=127.0.0.1:52353
2024-11-19T00:07:49,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:335090x0, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T00:07:49,533 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33509-0x101085226ab0000 connected
2024-11-19T00:07:49,555 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:07:49,556 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:07:49,561 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:07:49,562 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9, hbase.cluster.distributed=false
2024-11-19T00:07:49,564 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T00:07:49,567 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33509
2024-11-19T00:07:49,567 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33509
2024-11-19T00:07:49,573 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33509
2024-11-19T00:07:49,588 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33509
2024-11-19T00:07:49,589 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33509
2024-11-19T00:07:49,614 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/9c732c6543ff:0 server-side Connection retries=45
2024-11-19T00:07:49,615 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:07:49,615 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T00:07:49,615 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T00:07:49,615 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:07:49,615 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T00:07:49,615 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-19T00:07:49,615 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T00:07:49,618 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46105
2024-11-19T00:07:49,619 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46105 connecting to ZooKeeper ensemble=127.0.0.1:52353
2024-11-19T00:07:49,620 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:07:49,623 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:07:49,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461050x0, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T00:07:49,631 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46105-0x101085226ab0001 connected
2024-11-19T00:07:49,631 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:07:49,631 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-19T00:07:49,633 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-19T00:07:49,634 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-19T00:07:49,635 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T00:07:49,637 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46105
2024-11-19T00:07:49,638 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46105
2024-11-19T00:07:49,639 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46105
2024-11-19T00:07:49,640 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46105
2024-11-19T00:07:49,640 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46105
2024-11-19T00:07:49,653 DEBUG [M:0;9c732c6543ff:33509 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9c732c6543ff:33509
2024-11-19T00:07:49,653 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9c732c6543ff,33509,1731974869524
2024-11-19T00:07:49,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:07:49,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:07:49,656 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9c732c6543ff,33509,1731974869524
2024-11-19T00:07:49,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-19T00:07:49,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:49,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:49,659 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-19T00:07:49,660 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9c732c6543ff,33509,1731974869524 from backup master directory
2024-11-19T00:07:49,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9c732c6543ff,33509,1731974869524
2024-11-19T00:07:49,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:07:49,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:07:49,661 WARN  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T00:07:49,661 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9c732c6543ff,33509,1731974869524
2024-11-19T00:07:49,668 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/hbase.id] with ID: 297eb071-b734-457f-8ff5-80ac7dc78701
2024-11-19T00:07:49,668 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/.tmp/hbase.id
2024-11-19T00:07:49,700 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38767 is added to blk_1073741826_1002 (size=42)
2024-11-19T00:07:49,700 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:49,701 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32833 is added to blk_1073741826_1002 (size=42)
2024-11-19T00:07:49,702 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/.tmp/hbase.id]:[hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/hbase.id]
2024-11-19T00:07:49,728 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:07:49,728 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-19T00:07:49,730 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-19T00:07:49,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:49,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:49,757 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:49,771 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:49,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32833 is added to blk_1073741827_1003 (size=196)
2024-11-19T00:07:49,774 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38767 is added to blk_1073741827_1003 (size=196)
2024-11-19T00:07:49,776 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T00:07:49,776 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-19T00:07:49,777 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:07:49,795 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32833 is added to blk_1073741828_1004 (size=1189)
2024-11-19T00:07:49,796 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38767 is added to blk_1073741828_1004 (size=1189)
2024-11-19T00:07:49,798 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store
2024-11-19T00:07:49,839 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32833 is added to blk_1073741829_1005 (size=34)
2024-11-19T00:07:49,841 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38767 is added to blk_1073741829_1005 (size=34)
2024-11-19T00:07:49,843 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:07:49,843 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T00:07:49,843 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:07:49,843 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:07:49,843 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T00:07:49,843 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:07:49,843 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:07:49,843 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731974869843Disabling compacts and flushes for region at 1731974869843Disabling writes for close at 1731974869843Writing region close event to WAL at 1731974869843Closed at 1731974869843
2024-11-19T00:07:49,845 WARN  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/.initializing
2024-11-19T00:07:49,845 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/WALs/9c732c6543ff,33509,1731974869524
2024-11-19T00:07:49,849 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C33509%2C1731974869524, suffix=, logDir=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/WALs/9c732c6543ff,33509,1731974869524, archiveDir=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/oldWALs, maxLogs=10
2024-11-19T00:07:49,850 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C33509%2C1731974869524.1731974869850
2024-11-19T00:07:49,874 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/WALs/9c732c6543ff,33509,1731974869524/9c732c6543ff%2C33509%2C1731974869524.1731974869850
2024-11-19T00:07:49,889 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44891:44891),(127.0.0.1/127.0.0.1:37281:37281)]
2024-11-19T00:07:49,897 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:07:49,898 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:07:49,898 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:07:49,898 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:07:49,900 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:07:49,902 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-19T00:07:49,902 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:49,903 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:07:49,903 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:07:49,905 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-19T00:07:49,905 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:49,905 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:07:49,906 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:07:49,907 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-19T00:07:49,907 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:49,908 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:07:49,908 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:07:49,910 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-19T00:07:49,910 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:49,910 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:07:49,910 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:07:49,911 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:07:49,912 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:07:49,914 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:07:49,914 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:07:49,915 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-19T00:07:49,916 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:07:49,927 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:07:49,928 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=812856, jitterRate=0.03360071778297424}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-19T00:07:49,929 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731974869898Initializing all the Stores at 1731974869899 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974869899Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974869900 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974869900Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974869900Cleaning up temporary data from old regions at 1731974869914 (+14 ms)Region opened successfully at 1731974869929 (+15 ms)
2024-11-19T00:07:49,929 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-19T00:07:49,934 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c87c0f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0
2024-11-19T00:07:49,936 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-19T00:07:49,936 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-19T00:07:49,936 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-19T00:07:49,936 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-19T00:07:49,937 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-19T00:07:49,937 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-19T00:07:49,937 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-19T00:07:49,943 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-19T00:07:49,944 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-19T00:07:49,946 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-19T00:07:49,946 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-19T00:07:49,947 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-19T00:07:49,948 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-19T00:07:49,949 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-19T00:07:49,954 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-19T00:07:49,956 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-19T00:07:49,957 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-19T00:07:49,964 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-19T00:07:49,967 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-19T00:07:49,969 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-19T00:07:49,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T00:07:49,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:49,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T00:07:49,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:49,973 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9c732c6543ff,33509,1731974869524, sessionid=0x101085226ab0000, setting cluster-up flag (Was=false)
2024-11-19T00:07:49,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:49,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:49,983 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-19T00:07:49,993 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,33509,1731974869524
2024-11-19T00:07:49,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:49,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:50,003 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-19T00:07:50,004 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,33509,1731974869524
2024-11-19T00:07:50,012 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-19T00:07:50,014 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-19T00:07:50,014 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-19T00:07:50,015 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-19T00:07:50,015 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9c732c6543ff,33509,1731974869524
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-19T00:07:50,016 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:07:50,016 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:07:50,016 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:07:50,016 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:07:50,016 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9c732c6543ff:0, corePoolSize=10, maxPoolSize=10
2024-11-19T00:07:50,016 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:50,017 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2
2024-11-19T00:07:50,017 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:50,024 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731974900024
2024-11-19T00:07:50,024 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-19T00:07:50,024 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-19T00:07:50,024 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-19T00:07:50,024 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-19T00:07:50,024 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-19T00:07:50,024 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-19T00:07:50,025 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,025 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:07:50,025 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-19T00:07:50,025 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-19T00:07:50,026 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-19T00:07:50,026 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-19T00:07:50,026 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-19T00:07:50,026 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-19T00:07:50,027 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:50,027 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-19T00:07:50,029 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974870026,5,FailOnTimeoutGroup]
2024-11-19T00:07:50,033 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974870029,5,FailOnTimeoutGroup]
2024-11-19T00:07:50,033 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,033 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-19T00:07:50,033 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,033 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,046 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32833 is added to blk_1073741831_1007 (size=1321)
2024-11-19T00:07:50,047 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.HRegionServer(746): ClusterId : 297eb071-b734-457f-8ff5-80ac7dc78701
2024-11-19T00:07:50,047 DEBUG [RS:0;9c732c6543ff:46105 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-19T00:07:50,050 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38767 is added to blk_1073741831_1007 (size=1321)
2024-11-19T00:07:50,050 DEBUG [RS:0;9c732c6543ff:46105 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-19T00:07:50,051 DEBUG [RS:0;9c732c6543ff:46105 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-19T00:07:50,051 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-19T00:07:50,052 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9
2024-11-19T00:07:50,053 DEBUG [RS:0;9c732c6543ff:46105 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-19T00:07:50,054 DEBUG [RS:0;9c732c6543ff:46105 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e1d55f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0
2024-11-19T00:07:50,069 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32833 is added to blk_1073741832_1008 (size=32)
2024-11-19T00:07:50,070 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38767 is added to blk_1073741832_1008 (size=32)
2024-11-19T00:07:50,072 DEBUG [RS:0;9c732c6543ff:46105 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9c732c6543ff:46105
2024-11-19T00:07:50,073 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-19T00:07:50,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:07:50,073 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-19T00:07:50,073 DEBUG [RS:0;9c732c6543ff:46105 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-19T00:07:50,074 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.HRegionServer(2659): reportForDuty to master=9c732c6543ff,33509,1731974869524 with port=46105, startcode=1731974869614
2024-11-19T00:07:50,074 DEBUG [RS:0;9c732c6543ff:46105 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-19T00:07:50,074 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T00:07:50,076 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T00:07:50,076 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:50,076 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:07:50,077 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T00:07:50,078 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T00:07:50,078 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:50,078 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52279, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-11-19T00:07:50,078 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:07:50,079 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T00:07:50,079 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33509 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9c732c6543ff,46105,1731974869614
2024-11-19T00:07:50,079 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33509 {}] master.ServerManager(517): Registering regionserver=9c732c6543ff,46105,1731974869614
2024-11-19T00:07:50,080 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T00:07:50,080 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:50,081 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:07:50,081 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T00:07:50,082 DEBUG [RS:0;9c732c6543ff:46105 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9
2024-11-19T00:07:50,082 DEBUG [RS:0;9c732c6543ff:46105 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37953
2024-11-19T00:07:50,082 DEBUG [RS:0;9c732c6543ff:46105 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-19T00:07:50,084 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T00:07:50,085 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:50,085 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:07:50,085 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T00:07:50,086 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740
2024-11-19T00:07:50,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T00:07:50,087 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740
2024-11-19T00:07:50,088 DEBUG [RS:0;9c732c6543ff:46105 {}] zookeeper.ZKUtil(111): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9c732c6543ff,46105,1731974869614
2024-11-19T00:07:50,089 WARN  [RS:0;9c732c6543ff:46105 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T00:07:50,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T00:07:50,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T00:07:50,089 INFO  [RS:0;9c732c6543ff:46105 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:07:50,089 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T00:07:50,090 DEBUG [RS:0;9c732c6543ff:46105 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614
2024-11-19T00:07:50,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T00:07:50,095 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9c732c6543ff,46105,1731974869614]
2024-11-19T00:07:50,099 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-19T00:07:50,101 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:07:50,102 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-19T00:07:50,102 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=726461, jitterRate=-0.0762576013803482}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T00:07:50,103 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731974870073Initializing all the Stores at 1731974870074 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974870074Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974870074Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974870074Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974870074Cleaning up temporary data from old regions at 1731974870089 (+15 ms)Region opened successfully at 1731974870103 (+14 ms)
2024-11-19T00:07:50,103 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T00:07:50,103 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T00:07:50,103 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T00:07:50,103 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T00:07:50,103 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T00:07:50,105 INFO  [RS:0;9c732c6543ff:46105 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T00:07:50,105 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,108 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T00:07:50,109 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-19T00:07:50,109 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731974870103Disabling compacts and flushes for region at 1731974870103Disabling writes for close at 1731974870103Writing region close event to WAL at 1731974870108 (+5 ms)Closed at 1731974870108
2024-11-19T00:07:50,111 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:07:50,111 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-19T00:07:50,111 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-19T00:07:50,113 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T00:07:50,114 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-19T00:07:50,115 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-19T00:07:50,115 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,115 DEBUG [RS:0;9c732c6543ff:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:50,115 DEBUG [RS:0;9c732c6543ff:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:50,116 DEBUG [RS:0;9c732c6543ff:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:50,116 DEBUG [RS:0;9c732c6543ff:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:50,116 DEBUG [RS:0;9c732c6543ff:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:50,116 DEBUG [RS:0;9c732c6543ff:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2
2024-11-19T00:07:50,116 DEBUG [RS:0;9c732c6543ff:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:50,116 DEBUG [RS:0;9c732c6543ff:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:50,116 DEBUG [RS:0;9c732c6543ff:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:50,116 DEBUG [RS:0;9c732c6543ff:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:50,116 DEBUG [RS:0;9c732c6543ff:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:50,116 DEBUG [RS:0;9c732c6543ff:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:07:50,116 DEBUG [RS:0;9c732c6543ff:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3
2024-11-19T00:07:50,116 DEBUG [RS:0;9c732c6543ff:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3
2024-11-19T00:07:50,118 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,118 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,118 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,118 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,118 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,118 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,46105,1731974869614-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T00:07:50,137 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-19T00:07:50,138 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,46105,1731974869614-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,138 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,138 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.Replication(171): 9c732c6543ff,46105,1731974869614 started
2024-11-19T00:07:50,157 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,157 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.HRegionServer(1482): Serving as 9c732c6543ff,46105,1731974869614, RpcServer on 9c732c6543ff/172.17.0.2:46105, sessionid=0x101085226ab0001
2024-11-19T00:07:50,157 DEBUG [RS:0;9c732c6543ff:46105 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-19T00:07:50,157 DEBUG [RS:0;9c732c6543ff:46105 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9c732c6543ff,46105,1731974869614
2024-11-19T00:07:50,157 DEBUG [RS:0;9c732c6543ff:46105 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,46105,1731974869614'
2024-11-19T00:07:50,157 DEBUG [RS:0;9c732c6543ff:46105 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-19T00:07:50,158 DEBUG [RS:0;9c732c6543ff:46105 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-19T00:07:50,159 DEBUG [RS:0;9c732c6543ff:46105 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-19T00:07:50,159 DEBUG [RS:0;9c732c6543ff:46105 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-19T00:07:50,159 DEBUG [RS:0;9c732c6543ff:46105 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9c732c6543ff,46105,1731974869614
2024-11-19T00:07:50,159 DEBUG [RS:0;9c732c6543ff:46105 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,46105,1731974869614'
2024-11-19T00:07:50,159 DEBUG [RS:0;9c732c6543ff:46105 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-19T00:07:50,159 DEBUG [RS:0;9c732c6543ff:46105 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-19T00:07:50,160 DEBUG [RS:0;9c732c6543ff:46105 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-19T00:07:50,160 INFO  [RS:0;9c732c6543ff:46105 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-19T00:07:50,160 INFO  [RS:0;9c732c6543ff:46105 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-19T00:07:50,262 INFO  [RS:0;9c732c6543ff:46105 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C46105%2C1731974869614, suffix=, logDir=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614, archiveDir=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/oldWALs, maxLogs=32
2024-11-19T00:07:50,263 INFO  [RS:0;9c732c6543ff:46105 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C46105%2C1731974869614.1731974870263
2024-11-19T00:07:50,266 WARN  [9c732c6543ff:33509 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-19T00:07:50,270 INFO  [RS:0;9c732c6543ff:46105 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974870263
2024-11-19T00:07:50,276 DEBUG [RS:0;9c732c6543ff:46105 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:44891:44891)]
2024-11-19T00:07:50,516 DEBUG [9c732c6543ff:33509 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-19T00:07:50,517 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9c732c6543ff,46105,1731974869614
2024-11-19T00:07:50,518 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,46105,1731974869614, state=OPENING
2024-11-19T00:07:50,520 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-19T00:07:50,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:50,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:07:50,522 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T00:07:50,522 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:07:50,522 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:07:50,522 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,46105,1731974869614}]
2024-11-19T00:07:50,675 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-19T00:07:50,677 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34689, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-19T00:07:50,681 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-19T00:07:50,681 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:07:50,683 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C46105%2C1731974869614.meta, suffix=.meta, logDir=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614, archiveDir=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/oldWALs, maxLogs=32
2024-11-19T00:07:50,683 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C46105%2C1731974869614.meta.1731974870683.meta
2024-11-19T00:07:50,690 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.meta.1731974870683.meta
2024-11-19T00:07:50,691 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44891:44891),(127.0.0.1/127.0.0.1:37281:37281)]
2024-11-19T00:07:50,692 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:07:50,692 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-19T00:07:50,692 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-19T00:07:50,692 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-19T00:07:50,692 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-19T00:07:50,692 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:07:50,692 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-19T00:07:50,692 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-19T00:07:50,694 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T00:07:50,695 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T00:07:50,695 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:50,695 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:07:50,695 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T00:07:50,696 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T00:07:50,696 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:50,696 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:07:50,697 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T00:07:50,697 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T00:07:50,697 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:50,698 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:07:50,698 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T00:07:50,698 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T00:07:50,698 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:50,699 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:07:50,699 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T00:07:50,700 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740
2024-11-19T00:07:50,701 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740
2024-11-19T00:07:50,701 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:50,702 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T00:07:50,702 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T00:07:50,703 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T00:07:50,704 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T00:07:50,705 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=813556, jitterRate=0.034489989280700684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T00:07:50,705 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-19T00:07:50,706 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731974870692Writing region info on filesystem at 1731974870692Initializing all the Stores at 1731974870693 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974870693Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974870694 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974870694Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974870694Cleaning up temporary data from old regions at 1731974870702 (+8 ms)Running coprocessor post-open hooks at 1731974870705 (+3 ms)Region opened successfully at 1731974870706 (+1 ms)
2024-11-19T00:07:50,707 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731974870675
2024-11-19T00:07:50,709 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-19T00:07:50,709 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-19T00:07:50,710 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,46105,1731974869614
2024-11-19T00:07:50,711 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,46105,1731974869614, state=OPEN
2024-11-19T00:07:50,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T00:07:50,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T00:07:50,716 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9c732c6543ff,46105,1731974869614
2024-11-19T00:07:50,716 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:07:50,716 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:07:50,719 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-19T00:07:50,719 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,46105,1731974869614 in 194 msec
2024-11-19T00:07:50,722 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-19T00:07:50,722 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 608 msec
2024-11-19T00:07:50,722 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:07:50,722 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-19T00:07:50,724 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T00:07:50,724 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,46105,1731974869614, seqNum=-1]
2024-11-19T00:07:50,724 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T00:07:50,725 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48105, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T00:07:50,730 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 717 msec
2024-11-19T00:07:50,731 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731974870730, completionTime=-1
2024-11-19T00:07:50,731 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-19T00:07:50,731 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-19T00:07:50,732 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-19T00:07:50,732 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731974930732
2024-11-19T00:07:50,732 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731974990732
2024-11-19T00:07:50,732 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-11-19T00:07:50,733 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,33509,1731974869524-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,733 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,33509,1731974869524-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,733 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,33509,1731974869524-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,733 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9c732c6543ff:33509, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,733 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,733 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,735 DEBUG [master/9c732c6543ff:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-19T00:07:50,737 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.076sec
2024-11-19T00:07:50,737 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-19T00:07:50,737 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-19T00:07:50,737 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-19T00:07:50,737 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-19T00:07:50,737 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-19T00:07:50,737 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,33509,1731974869524-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T00:07:50,737 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,33509,1731974869524-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-19T00:07:50,740 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-19T00:07:50,740 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-19T00:07:50,740 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,33509,1731974869524-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:07:50,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67332887, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:07:50,746 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9c732c6543ff,33509,-1 for getting cluster id
2024-11-19T00:07:50,746 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-19T00:07:50,748 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '297eb071-b734-457f-8ff5-80ac7dc78701'
2024-11-19T00:07:50,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-19T00:07:50,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "297eb071-b734-457f-8ff5-80ac7dc78701"

2024-11-19T00:07:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c561d09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:07:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9c732c6543ff,33509,-1]
2024-11-19T00:07:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-19T00:07:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:07:50,751 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-19T00:07:50,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e9dc43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:07:50,752 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T00:07:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,46105,1731974869614, seqNum=-1]
2024-11-19T00:07:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T00:07:50,755 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T00:07:50,756 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9c732c6543ff,33509,1731974869524
2024-11-19T00:07:50,756 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:07:50,758 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:50,759 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-19T00:07:50,759 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-11-19T00:07:50,759 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-11-19T00:07:50,759 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-19T00:07:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is 9c732c6543ff,33509,1731974869524
2024-11-19T00:07:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7a32864a
2024-11-19T00:07:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-19T00:07:50,762 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-19T00:07:50,762 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33509 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-19T00:07:50,762 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33509 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-19T00:07:50,763 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33509 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T00:07:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33509 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-11-19T00:07:50,765 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-11-19T00:07:50,766 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:50,766 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33509 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-11-19T00:07:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33509 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T00:07:50,767 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-19T00:07:50,771 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:50,773 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32833 is added to blk_1073741835_1011 (size=395)
2024-11-19T00:07:50,773 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38767 is added to blk_1073741835_1011 (size=395)
2024-11-19T00:07:50,776 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 28cb3b7dcf0fdc4568bb361cd5458244, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9
2024-11-19T00:07:50,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38767 is added to blk_1073741836_1012 (size=78)
2024-11-19T00:07:50,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32833 is added to blk_1073741836_1012 (size=78)
2024-11-19T00:07:50,784 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:07:50,784 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 28cb3b7dcf0fdc4568bb361cd5458244, disabling compactions & flushes
2024-11-19T00:07:50,784 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.
2024-11-19T00:07:50,784 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.
2024-11-19T00:07:50,784 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244. after waiting 0 ms
2024-11-19T00:07:50,784 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.
2024-11-19T00:07:50,784 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.
2024-11-19T00:07:50,784 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 28cb3b7dcf0fdc4568bb361cd5458244:
Waiting for close lock at 1731974870784Disabling compacts and flushes for region at 1731974870784Disabling writes for close at 1731974870784Writing region close event to WAL at 1731974870784Closed at 1731974870784
2024-11-19T00:07:50,786 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-11-19T00:07:50,786 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731974870786"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731974870786"}]},"ts":"1731974870786"}
2024-11-19T00:07:50,789 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-19T00:07:50,790 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-19T00:07:50,790 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731974870790"}]},"ts":"1731974870790"}
2024-11-19T00:07:50,792 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-11-19T00:07:50,793 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=28cb3b7dcf0fdc4568bb361cd5458244, ASSIGN}]
2024-11-19T00:07:50,794 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=28cb3b7dcf0fdc4568bb361cd5458244, ASSIGN
2024-11-19T00:07:50,795 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=28cb3b7dcf0fdc4568bb361cd5458244, ASSIGN; state=OFFLINE, location=9c732c6543ff,46105,1731974869614; forceNewPlan=false, retain=false
2024-11-19T00:07:50,946 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=28cb3b7dcf0fdc4568bb361cd5458244, regionState=OPENING, regionLocation=9c732c6543ff,46105,1731974869614
2024-11-19T00:07:50,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=28cb3b7dcf0fdc4568bb361cd5458244, ASSIGN because future has completed
2024-11-19T00:07:50,950 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28cb3b7dcf0fdc4568bb361cd5458244, server=9c732c6543ff,46105,1731974869614}]
2024-11-19T00:07:51,107 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.
2024-11-19T00:07:51,107 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 28cb3b7dcf0fdc4568bb361cd5458244, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:07:51,107 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 28cb3b7dcf0fdc4568bb361cd5458244
2024-11-19T00:07:51,108 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:07:51,108 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 28cb3b7dcf0fdc4568bb361cd5458244
2024-11-19T00:07:51,108 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 28cb3b7dcf0fdc4568bb361cd5458244
2024-11-19T00:07:51,109 INFO  [StoreOpener-28cb3b7dcf0fdc4568bb361cd5458244-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 28cb3b7dcf0fdc4568bb361cd5458244 
2024-11-19T00:07:51,111 INFO  [StoreOpener-28cb3b7dcf0fdc4568bb361cd5458244-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28cb3b7dcf0fdc4568bb361cd5458244 columnFamilyName info
2024-11-19T00:07:51,111 DEBUG [StoreOpener-28cb3b7dcf0fdc4568bb361cd5458244-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:07:51,111 INFO  [StoreOpener-28cb3b7dcf0fdc4568bb361cd5458244-1 {}] regionserver.HStore(327): Store=28cb3b7dcf0fdc4568bb361cd5458244/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:07:51,111 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 28cb3b7dcf0fdc4568bb361cd5458244
2024-11-19T00:07:51,112 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/default/TestLogRolling-testLogRollOnPipelineRestart/28cb3b7dcf0fdc4568bb361cd5458244
2024-11-19T00:07:51,112 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/default/TestLogRolling-testLogRollOnPipelineRestart/28cb3b7dcf0fdc4568bb361cd5458244
2024-11-19T00:07:51,113 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 28cb3b7dcf0fdc4568bb361cd5458244
2024-11-19T00:07:51,113 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 28cb3b7dcf0fdc4568bb361cd5458244
2024-11-19T00:07:51,114 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 28cb3b7dcf0fdc4568bb361cd5458244
2024-11-19T00:07:51,117 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/default/TestLogRolling-testLogRollOnPipelineRestart/28cb3b7dcf0fdc4568bb361cd5458244/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:07:51,117 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 28cb3b7dcf0fdc4568bb361cd5458244; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=835807, jitterRate=0.06278391182422638}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-19T00:07:51,117 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 28cb3b7dcf0fdc4568bb361cd5458244
2024-11-19T00:07:51,118 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 28cb3b7dcf0fdc4568bb361cd5458244:
Running coprocessor pre-open hook at 1731974871108Writing region info on filesystem at 1731974871108Initializing all the Stores at 1731974871109 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974871109Cleaning up temporary data from old regions at 1731974871113 (+4 ms)Running coprocessor post-open hooks at 1731974871117 (+4 ms)Region opened successfully at 1731974871118 (+1 ms)
2024-11-19T00:07:51,119 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244., pid=6, masterSystemTime=1731974871102
2024-11-19T00:07:51,121 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.
2024-11-19T00:07:51,121 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.
2024-11-19T00:07:51,123 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=28cb3b7dcf0fdc4568bb361cd5458244, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,46105,1731974869614
2024-11-19T00:07:51,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28cb3b7dcf0fdc4568bb361cd5458244, server=9c732c6543ff,46105,1731974869614 because future has completed
2024-11-19T00:07:51,129 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-19T00:07:51,129 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 28cb3b7dcf0fdc4568bb361cd5458244, server=9c732c6543ff,46105,1731974869614 in 176 msec
2024-11-19T00:07:51,132 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-19T00:07:51,132 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=28cb3b7dcf0fdc4568bb361cd5458244, ASSIGN in 337 msec
2024-11-19T00:07:51,133 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-19T00:07:51,134 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731974871133"}]},"ts":"1731974871133"}
2024-11-19T00:07:51,136 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-11-19T00:07:51,138 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-11-19T00:07:51,140 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 375 msec
2024-11-19T00:07:51,701 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:51,756 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T00:07:51,758 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:51,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:51,772 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:51,772 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:51,772 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:51,772 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:51,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:51,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:51,777 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:51,778 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:51,778 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:51,781 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:07:52,702 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:52,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:52,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:53,703 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:53,760 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:53,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:54,704 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:54,760 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:54,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:55,704 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:55,761 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:55,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:56,099 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-19T00:07:56,099 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-11-19T00:07:56,253 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-19T00:07:56,253 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-19T00:07:56,254 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-19T00:07:56,254 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-11-19T00:07:56,254 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T00:07:56,254 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-19T00:07:56,705 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:56,761 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:56,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:57,705 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:57,762 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:57,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:58,706 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:58,763 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:58,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:59,707 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:59,763 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:07:59,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:00,708 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:00,764 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:00,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:00,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33509 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T00:08:00,863 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-11-19T00:08:00,864 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-11-19T00:08:00,867 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-11-19T00:08:00,867 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.
2024-11-19T00:08:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244., hostname=9c732c6543ff,46105,1731974869614, seqNum=2]
2024-11-19T00:08:01,708 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:01,765 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:01,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:02,709 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:02,765 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:02,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:02,873 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974870263
2024-11-19T00:08:02,874 WARN  [ResponseProcessor for block BP-1256842488-172.17.0.2-1731974868755:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1256842488-172.17.0.2-1731974868755:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:02,874 WARN  [ResponseProcessor for block BP-1256842488-172.17.0.2-1731974868755:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1256842488-172.17.0.2-1731974868755:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:02,874 WARN  [ResponseProcessor for block BP-1256842488-172.17.0.2-1731974868755:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1256842488-172.17.0.2-1731974868755:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-1256842488-172.17.0.2-1731974868755:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:38767,DS-26ae9a1c-e803-469a-8dfe-b9cfac092529,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:02,874 WARN  [DataStreamer for file /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.meta.1731974870683.meta block BP-1256842488-172.17.0.2-1731974868755:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1256842488-172.17.0.2-1731974868755:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38767,DS-26ae9a1c-e803-469a-8dfe-b9cfac092529,DISK], DatanodeInfoWithStorage[127.0.0.1:32833,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38767,DS-26ae9a1c-e803-469a-8dfe-b9cfac092529,DISK]) is bad.
2024-11-19T00:08:02,874 WARN  [DataStreamer for file /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/WALs/9c732c6543ff,33509,1731974869524/9c732c6543ff%2C33509%2C1731974869524.1731974869850 block BP-1256842488-172.17.0.2-1731974868755:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1256842488-172.17.0.2-1731974868755:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38767,DS-26ae9a1c-e803-469a-8dfe-b9cfac092529,DISK], DatanodeInfoWithStorage[127.0.0.1:32833,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38767,DS-26ae9a1c-e803-469a-8dfe-b9cfac092529,DISK]) is bad.
2024-11-19T00:08:02,875 WARN  [DataStreamer for file /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974870263 block BP-1256842488-172.17.0.2-1731974868755:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1256842488-172.17.0.2-1731974868755:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32833,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK], DatanodeInfoWithStorage[127.0.0.1:38767,DS-26ae9a1c-e803-469a-8dfe-b9cfac092529,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38767,DS-26ae9a1c-e803-469a-8dfe-b9cfac092529,DISK]) is bad.
2024-11-19T00:08:02,874 WARN  [PacketResponder: BP-1256842488-172.17.0.2-1731974868755:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:38767] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:02,875 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1221933714_22 at /127.0.0.1:40424 [Receiving block BP-1256842488-172.17.0.2-1731974868755:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:38767:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40424 dst: /127.0.0.1:38767
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:02,875 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1221933714_22 at /127.0.0.1:51552 [Receiving block BP-1256842488-172.17.0.2-1731974868755:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:32833:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51552 dst: /127.0.0.1:32833
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:02,875 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-387530927_22 at /127.0.0.1:40470 [Receiving block BP-1256842488-172.17.0.2-1731974868755:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:38767:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40470 dst: /127.0.0.1:38767
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:02,876 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-387530927_22 at /127.0.0.1:40452 [Receiving block BP-1256842488-172.17.0.2-1731974868755:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:38767:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40452 dst: /127.0.0.1:38767
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:02,875 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-387530927_22 at /127.0.0.1:51578 [Receiving block BP-1256842488-172.17.0.2-1731974868755:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:32833:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51578 dst: /127.0.0.1:32833
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:02,876 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-387530927_22 at /127.0.0.1:51572 [Receiving block BP-1256842488-172.17.0.2-1731974868755:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:32833:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51572 dst: /127.0.0.1:32833
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:02,877 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1fc8bed8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:08:02,877 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@44bd8b73{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:08:02,877 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:08:02,877 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a9fc515{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:08:02,877 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38cf0a15{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.log.dir/,STOPPED}
2024-11-19T00:08:02,879 WARN  [BP-1256842488-172.17.0.2-1731974868755 heartbeating to localhost/127.0.0.1:37953 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:08:02,879 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:08:02,879 WARN  [BP-1256842488-172.17.0.2-1731974868755 heartbeating to localhost/127.0.0.1:37953 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1256842488-172.17.0.2-1731974868755 (Datanode Uuid 03fef0cd-5312-4870-ab41-4af83df32bee) service to localhost/127.0.0.1:37953
2024-11-19T00:08:02,879 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:08:02,880 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/data/data3/current/BP-1256842488-172.17.0.2-1731974868755 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:08:02,880 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/data/data4/current/BP-1256842488-172.17.0.2-1731974868755 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:08:02,880 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:08:02,895 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:08:02,898 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:08:02,899 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:08:02,899 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:08:02,899 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T00:08:02,900 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371cd9e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:08:02,900 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5a0e55c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:08:03,013 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f330ba5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/java.io.tmpdir/jetty-localhost-34597-hadoop-hdfs-3_4_1-tests_jar-_-any-6315216551885955804/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:08:03,014 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4c49b856{HTTP/1.1, (http/1.1)}{localhost:34597}
2024-11-19T00:08:03,014 INFO  [Time-limited test {}] server.Server(415): Started @162021ms
2024-11-19T00:08:03,015 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:08:03,038 WARN  [ResponseProcessor for block BP-1256842488-172.17.0.2-1731974868755:blk_1073741834_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1256842488-172.17.0.2-1731974868755:blk_1073741834_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:03,038 WARN  [ResponseProcessor for block BP-1256842488-172.17.0.2-1731974868755:blk_1073741833_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1256842488-172.17.0.2-1731974868755:blk_1073741833_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:03,038 WARN  [ResponseProcessor for block BP-1256842488-172.17.0.2-1731974868755:blk_1073741830_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1256842488-172.17.0.2-1731974868755:blk_1073741830_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:03,039 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1221933714_22 at /127.0.0.1:43324 [Receiving block BP-1256842488-172.17.0.2-1731974868755:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:32833:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43324 dst: /127.0.0.1:32833
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:03,039 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-387530927_22 at /127.0.0.1:43332 [Receiving block BP-1256842488-172.17.0.2-1731974868755:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:32833:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43332 dst: /127.0.0.1:32833
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:03,039 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-387530927_22 at /127.0.0.1:43326 [Receiving block BP-1256842488-172.17.0.2-1731974868755:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:32833:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43326 dst: /127.0.0.1:32833
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:03,041 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@21865735{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:08:03,041 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@54562211{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:08:03,041 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:08:03,041 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fefca8b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:08:03,041 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cc14744{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.log.dir/,STOPPED}
2024-11-19T00:08:03,043 WARN  [BP-1256842488-172.17.0.2-1731974868755 heartbeating to localhost/127.0.0.1:37953 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:08:03,043 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:08:03,043 WARN  [BP-1256842488-172.17.0.2-1731974868755 heartbeating to localhost/127.0.0.1:37953 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1256842488-172.17.0.2-1731974868755 (Datanode Uuid ea9b0a6a-d45f-4c45-aa69-040e1f5651cd) service to localhost/127.0.0.1:37953
2024-11-19T00:08:03,043 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:08:03,044 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/data/data1/current/BP-1256842488-172.17.0.2-1731974868755 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:08:03,044 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/data/data2/current/BP-1256842488-172.17.0.2-1731974868755 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:08:03,044 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:08:03,055 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:08:03,058 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:08:03,058 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:08:03,058 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:08:03,058 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T00:08:03,059 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@343c572e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:08:03,059 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@542bdd12{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:08:03,103 WARN  [Thread-1333 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:08:03,105 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6ffcde689095f6b9 with lease ID 0x773b84388bc8ab55: from storage DS-26ae9a1c-e803-469a-8dfe-b9cfac092529 node DatanodeRegistration(127.0.0.1:41315, datanodeUuid=03fef0cd-5312-4870-ab41-4af83df32bee, infoPort=39223, infoSecurePort=0, ipcPort=42413, storageInfo=lv=-57;cid=testClusterID;nsid=615369472;c=1731974868755), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:08:03,106 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6ffcde689095f6b9 with lease ID 0x773b84388bc8ab55: from storage DS-9cc89262-61b5-46ed-8c03-ec3ce055a0f2 node DatanodeRegistration(127.0.0.1:41315, datanodeUuid=03fef0cd-5312-4870-ab41-4af83df32bee, infoPort=39223, infoSecurePort=0, ipcPort=42413, storageInfo=lv=-57;cid=testClusterID;nsid=615369472;c=1731974868755), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:08:03,175 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@686d6e83{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/java.io.tmpdir/jetty-localhost-39433-hadoop-hdfs-3_4_1-tests_jar-_-any-541862793399178138/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:08:03,175 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3a7e8687{HTTP/1.1, (http/1.1)}{localhost:39433}
2024-11-19T00:08:03,175 INFO  [Time-limited test {}] server.Server(415): Started @162183ms
2024-11-19T00:08:03,177 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:08:03,257 WARN  [Thread-1364 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:08:03,259 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x156884d3e715912b with lease ID 0x773b84388bc8ab56: from storage DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac node DatanodeRegistration(127.0.0.1:45867, datanodeUuid=ea9b0a6a-d45f-4c45-aa69-040e1f5651cd, infoPort=33071, infoSecurePort=0, ipcPort=34229, storageInfo=lv=-57;cid=testClusterID;nsid=615369472;c=1731974868755), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:08:03,259 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x156884d3e715912b with lease ID 0x773b84388bc8ab56: from storage DS-5e6cb786-f3a8-4e4b-ab57-eb625a535996 node DatanodeRegistration(127.0.0.1:45867, datanodeUuid=ea9b0a6a-d45f-4c45-aa69-040e1f5651cd, infoPort=33071, infoSecurePort=0, ipcPort=34229, storageInfo=lv=-57;cid=testClusterID;nsid=615369472;c=1731974868755), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:08:03,709 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:03,766 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:03,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:04,195 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-11-19T00:08:04,198 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-11-19T00:08:04,199 ERROR [FSHLog-0-hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9-prefix:9c732c6543ff,46105,1731974869614 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32833,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:04,199 WARN  [FSHLog-0-hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9-prefix:9c732c6543ff,46105,1731974869614 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32833,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:04,199 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9c732c6543ff%2C46105%2C1731974869614:(num 1731974870263) roll requested
2024-11-19T00:08:04,200 INFO  [regionserver/9c732c6543ff:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C46105%2C1731974869614.1731974884200
2024-11-19T00:08:04,206 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974870263 newFile=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974884200
2024-11-19T00:08:04,206 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:04,206 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:04,206 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:04,206 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:04,207 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:04,207 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974870263 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974884200
2024-11-19T00:08:04,207 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32833,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:04,207 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32833,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:04,207 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974870263
2024-11-19T00:08:04,208 WARN  [IPC Server handler 2 on default port 37953 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974870263 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1015
2024-11-19T00:08:04,208 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974870263 after 1ms
2024-11-19T00:08:04,209 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33071:33071),(127.0.0.1/127.0.0.1:39223:39223)]
2024-11-19T00:08:04,209 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974870263 is not closed yet, will try archiving it next time
2024-11-19T00:08:04,260 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45867 is added to blk_1073741833_1017 (size=1632)
2024-11-19T00:08:04,710 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:04,766 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:04,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:05,711 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:05,767 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:05,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:06,213 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-11-19T00:08:06,711 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:06,767 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:06,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:07,105 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1015: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-19T00:08:07,712 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:07,768 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:07,780 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:08,209 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974870263 after 4002ms
2024-11-19T00:08:08,216 WARN  [ResponseProcessor for block BP-1256842488-172.17.0.2-1731974868755:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1256842488-172.17.0.2-1731974868755:blk_1073741837_1016
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:08,216 WARN  [DataStreamer for file /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974884200 block BP-1256842488-172.17.0.2-1731974868755:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1256842488-172.17.0.2-1731974868755:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45867,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK], DatanodeInfoWithStorage[127.0.0.1:41315,DS-26ae9a1c-e803-469a-8dfe-b9cfac092529,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45867,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK]) is bad.
2024-11-19T00:08:08,216 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-387530927_22 at /127.0.0.1:44958 [Receiving block BP-1256842488-172.17.0.2-1731974868755:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:41315:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:44958 dst: /127.0.0.1:41315
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:08,216 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-387530927_22 at /127.0.0.1:60588 [Receiving block BP-1256842488-172.17.0.2-1731974868755:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:45867:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60588 dst: /127.0.0.1:45867
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:08,218 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@686d6e83{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:08:08,218 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3a7e8687{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:08:08,218 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:08:08,219 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@542bdd12{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:08:08,219 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@343c572e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.log.dir/,STOPPED}
2024-11-19T00:08:08,220 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:08:08,220 WARN  [BP-1256842488-172.17.0.2-1731974868755 heartbeating to localhost/127.0.0.1:37953 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:08:08,220 WARN  [BP-1256842488-172.17.0.2-1731974868755 heartbeating to localhost/127.0.0.1:37953 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1256842488-172.17.0.2-1731974868755 (Datanode Uuid ea9b0a6a-d45f-4c45-aa69-040e1f5651cd) service to localhost/127.0.0.1:37953
2024-11-19T00:08:08,220 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:08:08,221 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/data/data1/current/BP-1256842488-172.17.0.2-1731974868755 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:08:08,221 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/data/data2/current/BP-1256842488-172.17.0.2-1731974868755 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:08:08,221 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:08:08,231 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:08:08,235 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:08:08,236 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:08:08,236 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:08:08,236 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T00:08:08,236 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5a382d25{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:08:08,237 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@570cb725{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:08:08,350 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@49d7084d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/java.io.tmpdir/jetty-localhost-38885-hadoop-hdfs-3_4_1-tests_jar-_-any-17285548164622996775/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:08:08,350 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@731665df{HTTP/1.1, (http/1.1)}{localhost:38885}
2024-11-19T00:08:08,350 INFO  [Time-limited test {}] server.Server(415): Started @167357ms
2024-11-19T00:08:08,351 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:08:08,370 WARN  [ResponseProcessor for block BP-1256842488-172.17.0.2-1731974868755:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1256842488-172.17.0.2-1731974868755:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:08,370 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-387530927_22 at /127.0.0.1:44978 [Receiving block BP-1256842488-172.17.0.2-1731974868755:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:41315:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:44978 dst: /127.0.0.1:41315
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:08,373 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f330ba5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:08:08,373 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4c49b856{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:08:08,373 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:08:08,374 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5a0e55c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:08:08,374 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371cd9e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.log.dir/,STOPPED}
2024-11-19T00:08:08,375 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:08:08,375 WARN  [BP-1256842488-172.17.0.2-1731974868755 heartbeating to localhost/127.0.0.1:37953 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:08:08,375 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:08:08,375 WARN  [BP-1256842488-172.17.0.2-1731974868755 heartbeating to localhost/127.0.0.1:37953 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1256842488-172.17.0.2-1731974868755 (Datanode Uuid 03fef0cd-5312-4870-ab41-4af83df32bee) service to localhost/127.0.0.1:37953
2024-11-19T00:08:08,376 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/data/data3/current/BP-1256842488-172.17.0.2-1731974868755 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:08:08,376 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/data/data4/current/BP-1256842488-172.17.0.2-1731974868755 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:08:08,376 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:08:08,399 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:08:08,402 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:08:08,404 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:08:08,404 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:08:08,404 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T00:08:08,405 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5091fc79{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:08:08,405 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7da73466{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:08:08,438 WARN  [Thread-1407 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:08:08,441 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x50f0ba5bbea0a29 with lease ID 0x773b84388bc8ab57: from storage DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac node DatanodeRegistration(127.0.0.1:38307, datanodeUuid=ea9b0a6a-d45f-4c45-aa69-040e1f5651cd, infoPort=42867, infoSecurePort=0, ipcPort=44485, storageInfo=lv=-57;cid=testClusterID;nsid=615369472;c=1731974868755), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-19T00:08:08,441 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x50f0ba5bbea0a29 with lease ID 0x773b84388bc8ab57: from storage DS-5e6cb786-f3a8-4e4b-ab57-eb625a535996 node DatanodeRegistration(127.0.0.1:38307, datanodeUuid=ea9b0a6a-d45f-4c45-aa69-040e1f5651cd, infoPort=42867, infoSecurePort=0, ipcPort=44485, storageInfo=lv=-57;cid=testClusterID;nsid=615369472;c=1731974868755), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:08:08,523 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@428b2957{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/java.io.tmpdir/jetty-localhost-37081-hadoop-hdfs-3_4_1-tests_jar-_-any-13105733185140233967/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:08:08,524 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@bc30fa1{HTTP/1.1, (http/1.1)}{localhost:37081}
2024-11-19T00:08:08,524 INFO  [Time-limited test {}] server.Server(415): Started @167531ms
2024-11-19T00:08:08,525 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:08:08,603 WARN  [Thread-1438 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:08:08,606 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x73de9e992b14480a with lease ID 0x773b84388bc8ab58: from storage DS-26ae9a1c-e803-469a-8dfe-b9cfac092529 node DatanodeRegistration(127.0.0.1:44319, datanodeUuid=03fef0cd-5312-4870-ab41-4af83df32bee, infoPort=42143, infoSecurePort=0, ipcPort=35055, storageInfo=lv=-57;cid=testClusterID;nsid=615369472;c=1731974868755), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:08:08,606 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x73de9e992b14480a with lease ID 0x773b84388bc8ab58: from storage DS-9cc89262-61b5-46ed-8c03-ec3ce055a0f2 node DatanodeRegistration(127.0.0.1:44319, datanodeUuid=03fef0cd-5312-4870-ab41-4af83df32bee, infoPort=42143, infoSecurePort=0, ipcPort=35055, storageInfo=lv=-57;cid=testClusterID;nsid=615369472;c=1731974868755), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-19T00:08:08,712 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:08,769 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:08,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:09,543 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-11-19T00:08:09,545 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-11-19T00:08:09,547 ERROR [FSHLog-0-hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9-prefix:9c732c6543ff,46105,1731974869614 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41315,DS-26ae9a1c-e803-469a-8dfe-b9cfac092529,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:09,547 WARN  [FSHLog-0-hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9-prefix:9c732c6543ff,46105,1731974869614 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41315,DS-26ae9a1c-e803-469a-8dfe-b9cfac092529,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:09,547 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9c732c6543ff%2C46105%2C1731974869614:(num 1731974884200) roll requested
2024-11-19T00:08:09,547 INFO  [regionserver/9c732c6543ff:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C46105%2C1731974869614.1731974889547
2024-11-19T00:08:09,558 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974884200 newFile=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974889547
2024-11-19T00:08:09,558 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:09,559 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:09,559 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:09,559 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:09,559 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:09,559 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974884200 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974889547
2024-11-19T00:08:09,559 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41315,DS-26ae9a1c-e803-469a-8dfe-b9cfac092529,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:09,559 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41315,DS-26ae9a1c-e803-469a-8dfe-b9cfac092529,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:09,560 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974884200
2024-11-19T00:08:09,560 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42143:42143),(127.0.0.1/127.0.0.1:42867:42867)]
2024-11-19T00:08:09,560 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974884200 is not closed yet, will try archiving it next time
2024-11-19T00:08:09,560 WARN  [IPC Server handler 1 on default port 37953 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974884200 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-11-19T00:08:09,560 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974884200 after 0ms
2024-11-19T00:08:09,713 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:09,769 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:09,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:10,714 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:10,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:10,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:11,562 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C46105%2C1731974869614.1731974891561
2024-11-19T00:08:11,567 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974889547 newFile=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974891561
2024-11-19T00:08:11,567 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:11,567 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:11,568 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:11,568 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:11,568 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:11,568 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974889547 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974891561
2024-11-19T00:08:11,569 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42867:42867),(127.0.0.1/127.0.0.1:42143:42143)]
2024-11-19T00:08:11,569 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974884200 is not closed yet, will try archiving it next time
2024-11-19T00:08:11,569 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974889547 is not closed yet, will try archiving it next time
2024-11-19T00:08:11,569 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974870263
2024-11-19T00:08:11,569 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974870263
2024-11-19T00:08:11,570 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38307 is added to blk_1073741838_1019 (size=1264)
2024-11-19T00:08:11,570 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44319 is added to blk_1073741838_1019 (size=1264)
2024-11-19T00:08:11,570 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974870263 after 1ms
2024-11-19T00:08:11,570 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974870263
2024-11-19T00:08:11,571 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974884200 is not closed yet, will try archiving it next time
2024-11-19T00:08:11,580 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1731974871118/Put/vlen=218/seqid=0]
2024-11-19T00:08:11,580 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1731974880871/Put/vlen=1045/seqid=0]
2024-11-19T00:08:11,580 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974870263
2024-11-19T00:08:11,580 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974884200
2024-11-19T00:08:11,580 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974884200
2024-11-19T00:08:11,581 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974884200 after 1ms
2024-11-19T00:08:11,581 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974884200
2024-11-19T00:08:11,585 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1731974884199/Put/vlen=1045/seqid=0]
2024-11-19T00:08:11,585 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1731974886214/Put/vlen=1045/seqid=0]
2024-11-19T00:08:11,585 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974884200
2024-11-19T00:08:11,585 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974889547
2024-11-19T00:08:11,585 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974889547
2024-11-19T00:08:11,585 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974889547 after 0ms
2024-11-19T00:08:11,585 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974889547
2024-11-19T00:08:11,588 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1731974889546/Put/vlen=1045/seqid=0]
2024-11-19T00:08:11,588 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974891561
2024-11-19T00:08:11,588 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974891561
2024-11-19T00:08:11,589 WARN  [IPC Server handler 0 on default port 37953 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974891561 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-11-19T00:08:11,589 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974891561 after 1ms
2024-11-19T00:08:11,714 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:11,771 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:11,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:12,442 WARN  [ResponseProcessor for block BP-1256842488-172.17.0.2-1731974868755:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1256842488-172.17.0.2-1731974868755:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:12,442 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1221933714_22 at /127.0.0.1:52612 [Receiving block BP-1256842488-172.17.0.2-1731974868755:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:38307:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52612 dst: /127.0.0.1:38307
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38307 remote=/127.0.0.1:52612]. Total timeout mills is 60000, 59125 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:12,442 WARN  [DataStreamer for file /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974891561 block BP-1256842488-172.17.0.2-1731974868755:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1256842488-172.17.0.2-1731974868755:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38307,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK], DatanodeInfoWithStorage[127.0.0.1:44319,DS-26ae9a1c-e803-469a-8dfe-b9cfac092529,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38307,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK]) is bad.
2024-11-19T00:08:12,442 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1221933714_22 at /127.0.0.1:59204 [Receiving block BP-1256842488-172.17.0.2-1731974868755:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:44319:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59204 dst: /127.0.0.1:44319
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:12,443 WARN  [DataStreamer for file /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974891561 block BP-1256842488-172.17.0.2-1731974868755:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1256842488-172.17.0.2-1731974868755:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:12,451 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38307 is added to blk_1073741839_1022 (size=85)
2024-11-19T00:08:12,454 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44319 is added to blk_1073741839_1022 (size=85)
2024-11-19T00:08:12,715 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:12,771 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:12,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:13,441 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-19T00:08:13,561 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974884200 after 4001ms
2024-11-19T00:08:13,716 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:13,772 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:13,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:14,716 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:14,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:14,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:15,590 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974891561 after 4002ms
2024-11-19T00:08:15,590 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974891561
2024-11-19T00:08:15,594 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974891561
2024-11-19T00:08:15,594 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-11-19T00:08:15,595 ERROR [FSHLog-0-hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9-prefix:9c732c6543ff,46105,1731974869614.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32833,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:15,595 WARN  [FSHLog-0-hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9-prefix:9c732c6543ff,46105,1731974869614.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32833,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:15,595 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9c732c6543ff%2C46105%2C1731974869614.meta:.meta(num 1731974870683) roll requested
2024-11-19T00:08:15,595 INFO  [regionserver/9c732c6543ff:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C46105%2C1731974869614.meta.1731974895595.meta
2024-11-19T00:08:15,600 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:15,600 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:15,601 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:15,601 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:15,601 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:15,601 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.meta.1731974870683.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.meta.1731974895595.meta
2024-11-19T00:08:15,601 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32833,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:15,601 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32833,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:15,601 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.meta.1731974870683.meta
2024-11-19T00:08:15,602 WARN  [IPC Server handler 2 on default port 37953 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.meta.1731974870683.meta has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741834_1014
2024-11-19T00:08:15,602 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.meta.1731974870683.meta after 0ms
2024-11-19T00:08:15,604 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42867:42867),(127.0.0.1/127.0.0.1:42143:42143)]
2024-11-19T00:08:15,604 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.meta.1731974870683.meta is not closed yet, will try archiving it next time
2024-11-19T00:08:15,620 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740/.tmp/info/f76217e4fa0b4546b58f8111e239dae3 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244./info:regioninfo/1731974871122/Put/seqid=0
2024-11-19T00:08:15,624 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44319 is added to blk_1073741841_1025 (size=7125)
2024-11-19T00:08:15,625 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38307 is added to blk_1073741841_1025 (size=7125)
2024-11-19T00:08:15,625 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740/.tmp/info/f76217e4fa0b4546b58f8111e239dae3
2024-11-19T00:08:15,645 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740/.tmp/ns/cc36d0119bad47d989de38fdbd069fb0 is 43, key is default/ns:d/1731974870726/Put/seqid=0
2024-11-19T00:08:15,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44319 is added to blk_1073741842_1026 (size=5153)
2024-11-19T00:08:15,650 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38307 is added to blk_1073741842_1026 (size=5153)
2024-11-19T00:08:15,650 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740/.tmp/ns/cc36d0119bad47d989de38fdbd069fb0
2024-11-19T00:08:15,669 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740/.tmp/table/531e9c5fdcee46ed8076a0ad33c6bd41 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1731974871133/Put/seqid=0
2024-11-19T00:08:15,674 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38307 is added to blk_1073741843_1027 (size=5438)
2024-11-19T00:08:15,674 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44319 is added to blk_1073741843_1027 (size=5438)
2024-11-19T00:08:15,675 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740/.tmp/table/531e9c5fdcee46ed8076a0ad33c6bd41
2024-11-19T00:08:15,680 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740/.tmp/info/f76217e4fa0b4546b58f8111e239dae3 as hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740/info/f76217e4fa0b4546b58f8111e239dae3
2024-11-19T00:08:15,685 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740/info/f76217e4fa0b4546b58f8111e239dae3, entries=10, sequenceid=11, filesize=7.0 K
2024-11-19T00:08:15,686 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740/.tmp/ns/cc36d0119bad47d989de38fdbd069fb0 as hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740/ns/cc36d0119bad47d989de38fdbd069fb0
2024-11-19T00:08:15,691 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740/ns/cc36d0119bad47d989de38fdbd069fb0, entries=2, sequenceid=11, filesize=5.0 K
2024-11-19T00:08:15,692 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740/.tmp/table/531e9c5fdcee46ed8076a0ad33c6bd41 as hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740/table/531e9c5fdcee46ed8076a0ad33c6bd41
2024-11-19T00:08:15,696 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740/table/531e9c5fdcee46ed8076a0ad33c6bd41, entries=2, sequenceid=11, filesize=5.3 K
2024-11-19T00:08:15,698 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 103ms, sequenceid=11, compaction requested=false
2024-11-19T00:08:15,698 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-19T00:08:15,698 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 28cb3b7dcf0fdc4568bb361cd5458244 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-19T00:08:15,698 ERROR [FSHLog-0-hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9-prefix:9c732c6543ff,46105,1731974869614 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1256842488-172.17.0.2-1731974868755:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:15,698 WARN  [FSHLog-0-hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9-prefix:9c732c6543ff,46105,1731974869614 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1256842488-172.17.0.2-1731974868755:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:15,699 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9c732c6543ff%2C46105%2C1731974869614:(num 1731974891561) roll requested
2024-11-19T00:08:15,699 INFO  [regionserver/9c732c6543ff:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C46105%2C1731974869614.1731974895699
2024-11-19T00:08:15,703 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974891561 newFile=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974895699
2024-11-19T00:08:15,704 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:15,704 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:15,704 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:15,704 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:15,704 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:15,704 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974891561 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974895699
2024-11-19T00:08:15,704 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1256842488-172.17.0.2-1731974868755:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:15,705 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1256842488-172.17.0.2-1731974868755:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:15,705 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974891561
2024-11-19T00:08:15,705 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42143:42143),(127.0.0.1/127.0.0.1:42867:42867)]
2024-11-19T00:08:15,705 DEBUG [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974891561 is not closed yet, will try archiving it next time
2024-11-19T00:08:15,705 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974891561 after 0ms
2024-11-19T00:08:15,706 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.1731974891561 to hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/oldWALs/9c732c6543ff%2C46105%2C1731974869614.1731974891561
2024-11-19T00:08:15,717 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:15,721 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/default/TestLogRolling-testLogRollOnPipelineRestart/28cb3b7dcf0fdc4568bb361cd5458244/.tmp/info/396ca2b5398d48bd88064f6b82d739be is 1080, key is row1002/info:/1731974880871/Put/seqid=0
2024-11-19T00:08:15,726 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44319 is added to blk_1073741845_1029 (size=9270)
2024-11-19T00:08:15,726 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38307 is added to blk_1073741845_1029 (size=9270)
2024-11-19T00:08:15,726 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/default/TestLogRolling-testLogRollOnPipelineRestart/28cb3b7dcf0fdc4568bb361cd5458244/.tmp/info/396ca2b5398d48bd88064f6b82d739be
2024-11-19T00:08:15,732 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/default/TestLogRolling-testLogRollOnPipelineRestart/28cb3b7dcf0fdc4568bb361cd5458244/.tmp/info/396ca2b5398d48bd88064f6b82d739be as hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/default/TestLogRolling-testLogRollOnPipelineRestart/28cb3b7dcf0fdc4568bb361cd5458244/info/396ca2b5398d48bd88064f6b82d739be
2024-11-19T00:08:15,737 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/default/TestLogRolling-testLogRollOnPipelineRestart/28cb3b7dcf0fdc4568bb361cd5458244/info/396ca2b5398d48bd88064f6b82d739be, entries=4, sequenceid=8, filesize=9.1 K
2024-11-19T00:08:15,738 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 28cb3b7dcf0fdc4568bb361cd5458244 in 40ms, sequenceid=8, compaction requested=false
2024-11-19T00:08:15,738 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 28cb3b7dcf0fdc4568bb361cd5458244:

2024-11-19T00:08:15,743 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-19T00:08:15,744 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T00:08:15,744 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:08:15,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:08:15,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:08:15,744 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-19T00:08:15,744 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-19T00:08:15,744 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=298236265, stopped=false
2024-11-19T00:08:15,744 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9c732c6543ff,33509,1731974869524
2024-11-19T00:08:15,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T00:08:15,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T00:08:15,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:08:15,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:08:15,746 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T00:08:15,746 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:08:15,747 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:08:15,747 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T00:08:15,747 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:08:15,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:08:15,747 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9c732c6543ff,46105,1731974869614' *****
2024-11-19T00:08:15,747 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-19T00:08:15,747 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-19T00:08:15,748 INFO  [RS:0;9c732c6543ff:46105 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-19T00:08:15,748 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-19T00:08:15,748 INFO  [RS:0;9c732c6543ff:46105 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-19T00:08:15,748 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.HRegionServer(3091): Received CLOSE for 28cb3b7dcf0fdc4568bb361cd5458244
2024-11-19T00:08:15,748 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.HRegionServer(959): stopping server 9c732c6543ff,46105,1731974869614
2024-11-19T00:08:15,748 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T00:08:15,748 INFO  [RS:0;9c732c6543ff:46105 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9c732c6543ff:46105.
2024-11-19T00:08:15,748 DEBUG [RS:0;9c732c6543ff:46105 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:08:15,748 DEBUG [RS:0;9c732c6543ff:46105 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:08:15,748 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 28cb3b7dcf0fdc4568bb361cd5458244, disabling compactions & flushes
2024-11-19T00:08:15,748 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.
2024-11-19T00:08:15,748 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-19T00:08:15,748 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-19T00:08:15,748 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.
2024-11-19T00:08:15,748 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-19T00:08:15,748 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244. after waiting 0 ms
2024-11-19T00:08:15,748 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-19T00:08:15,748 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.
2024-11-19T00:08:15,749 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-19T00:08:15,749 DEBUG [RS:0;9c732c6543ff:46105 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 28cb3b7dcf0fdc4568bb361cd5458244=TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.}
2024-11-19T00:08:15,749 DEBUG [RS:0;9c732c6543ff:46105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 28cb3b7dcf0fdc4568bb361cd5458244
2024-11-19T00:08:15,749 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T00:08:15,749 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T00:08:15,749 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T00:08:15,749 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T00:08:15,749 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T00:08:15,757 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/default/TestLogRolling-testLogRollOnPipelineRestart/28cb3b7dcf0fdc4568bb361cd5458244/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-11-19T00:08:15,758 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.
2024-11-19T00:08:15,758 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-19T00:08:15,758 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 28cb3b7dcf0fdc4568bb361cd5458244:
Waiting for close lock at 1731974895748Running coprocessor pre-close hooks at 1731974895748Disabling compacts and flushes for region at 1731974895748Disabling writes for close at 1731974895748Writing region close event to WAL at 1731974895753 (+5 ms)Running coprocessor post-close hooks at 1731974895758 (+5 ms)Closed at 1731974895758
2024-11-19T00:08:15,758 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731974870762.28cb3b7dcf0fdc4568bb361cd5458244.
2024-11-19T00:08:15,758 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T00:08:15,758 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T00:08:15,758 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731974895749Running coprocessor pre-close hooks at 1731974895749Disabling compacts and flushes for region at 1731974895749Disabling writes for close at 1731974895749Writing region close event to WAL at 1731974895754 (+5 ms)Running coprocessor post-close hooks at 1731974895758 (+4 ms)Closed at 1731974895758
2024-11-19T00:08:15,758 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-19T00:08:15,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:15,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:15,949 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.HRegionServer(976): stopping server 9c732c6543ff,46105,1731974869614; all regions closed.
2024-11-19T00:08:15,950 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:15,950 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:15,950 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:15,950 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:15,950 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:15,952 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44319 is added to blk_1073741840_1023 (size=825)
2024-11-19T00:08:15,952 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38307 is added to blk_1073741840_1023 (size=825)
2024-11-19T00:08:16,119 INFO  [regionserver/9c732c6543ff:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-19T00:08:16,119 INFO  [regionserver/9c732c6543ff:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-19T00:08:16,122 INFO  [regionserver/9c732c6543ff:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T00:08:16,253 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T00:08:16,253 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-19T00:08:16,253 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-19T00:08:16,717 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:16,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:16,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:17,718 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:17,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:17,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:18,718 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:18,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:18,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:19,499 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-19T00:08:19,603 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.meta.1731974870683.meta after 4001ms
2024-11-19T00:08:19,603 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/WALs/9c732c6543ff,46105,1731974869614/9c732c6543ff%2C46105%2C1731974869614.meta.1731974870683.meta to hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/oldWALs/9c732c6543ff%2C46105%2C1731974869614.meta.1731974870683.meta
2024-11-19T00:08:19,606 DEBUG [RS:0;9c732c6543ff:46105 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/oldWALs
2024-11-19T00:08:19,606 INFO  [RS:0;9c732c6543ff:46105 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9c732c6543ff%2C46105%2C1731974869614.meta:.meta(num 1731974895595)
2024-11-19T00:08:19,606 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:19,606 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:19,606 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:19,607 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:19,607 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:19,607 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1014: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-19T00:08:19,608 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38307 is added to blk_1073741844_1028 (size=1162)
2024-11-19T00:08:19,609 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44319 is added to blk_1073741844_1028 (size=1162)
2024-11-19T00:08:19,613 DEBUG [RS:0;9c732c6543ff:46105 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/oldWALs
2024-11-19T00:08:19,613 INFO  [RS:0;9c732c6543ff:46105 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9c732c6543ff%2C46105%2C1731974869614:(num 1731974895699)
2024-11-19T00:08:19,613 DEBUG [RS:0;9c732c6543ff:46105 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:08:19,613 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T00:08:19,613 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T00:08:19,613 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.ChoreService(370): Chore service for: regionserver/9c732c6543ff:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-19T00:08:19,613 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T00:08:19,613 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T00:08:19,613 INFO  [RS:0;9c732c6543ff:46105 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46105
2024-11-19T00:08:19,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9c732c6543ff,46105,1731974869614
2024-11-19T00:08:19,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T00:08:19,615 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T00:08:19,617 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9c732c6543ff,46105,1731974869614]
2024-11-19T00:08:19,618 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9c732c6543ff,46105,1731974869614 already deleted, retry=false
2024-11-19T00:08:19,618 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9c732c6543ff,46105,1731974869614 expired; onlineServers=0
2024-11-19T00:08:19,618 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9c732c6543ff,33509,1731974869524' *****
2024-11-19T00:08:19,618 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-19T00:08:19,619 INFO  [M:0;9c732c6543ff:33509 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T00:08:19,619 INFO  [M:0;9c732c6543ff:33509 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T00:08:19,619 DEBUG [M:0;9c732c6543ff:33509 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-19T00:08:19,619 DEBUG [M:0;9c732c6543ff:33509 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-19T00:08:19,619 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-19T00:08:19,619 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974870029 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974870029,5,FailOnTimeoutGroup]
2024-11-19T00:08:19,619 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974870026 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974870026,5,FailOnTimeoutGroup]
2024-11-19T00:08:19,619 INFO  [M:0;9c732c6543ff:33509 {}] hbase.ChoreService(370): Chore service for: master/9c732c6543ff:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-19T00:08:19,619 INFO  [M:0;9c732c6543ff:33509 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T00:08:19,619 DEBUG [M:0;9c732c6543ff:33509 {}] master.HMaster(1795): Stopping service threads
2024-11-19T00:08:19,619 INFO  [M:0;9c732c6543ff:33509 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-19T00:08:19,619 INFO  [M:0;9c732c6543ff:33509 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T00:08:19,619 INFO  [M:0;9c732c6543ff:33509 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-19T00:08:19,619 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-19T00:08:19,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-19T00:08:19,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:08:19,620 DEBUG [M:0;9c732c6543ff:33509 {}] zookeeper.ZKUtil(347): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-19T00:08:19,620 WARN  [M:0;9c732c6543ff:33509 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-19T00:08:19,621 INFO  [M:0;9c732c6543ff:33509 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/.lastflushedseqids
2024-11-19T00:08:19,626 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38307 is added to blk_1073741846_1030 (size=139)
2024-11-19T00:08:19,626 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44319 is added to blk_1073741846_1030 (size=139)
2024-11-19T00:08:19,626 INFO  [M:0;9c732c6543ff:33509 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-19T00:08:19,626 INFO  [M:0;9c732c6543ff:33509 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-19T00:08:19,627 DEBUG [M:0;9c732c6543ff:33509 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T00:08:19,627 INFO  [M:0;9c732c6543ff:33509 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:08:19,627 DEBUG [M:0;9c732c6543ff:33509 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:08:19,627 DEBUG [M:0;9c732c6543ff:33509 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T00:08:19,627 DEBUG [M:0;9c732c6543ff:33509 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:08:19,627 INFO  [M:0;9c732c6543ff:33509 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB
2024-11-19T00:08:19,627 ERROR [FSHLog-0-hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData-prefix:9c732c6543ff,33509,1731974869524 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32833,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:19,627 WARN  [FSHLog-0-hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData-prefix:9c732c6543ff,33509,1731974869524 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32833,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:19,627 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 9c732c6543ff%2C33509%2C1731974869524:(num 1731974869850) roll requested
2024-11-19T00:08:19,628 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C33509%2C1731974869524.1731974899627
2024-11-19T00:08:19,632 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:19,632 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:19,632 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:19,632 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:19,632 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:19,633 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/WALs/9c732c6543ff,33509,1731974869524/9c732c6543ff%2C33509%2C1731974869524.1731974869850 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/WALs/9c732c6543ff,33509,1731974869524/9c732c6543ff%2C33509%2C1731974869524.1731974899627
2024-11-19T00:08:19,633 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32833,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:19,633 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32833,DS-50a7a50f-e34d-42fa-b6d9-03c12f8a85ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T00:08:19,633 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/WALs/9c732c6543ff,33509,1731974869524/9c732c6543ff%2C33509%2C1731974869524.1731974869850
2024-11-19T00:08:19,633 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42143:42143),(127.0.0.1/127.0.0.1:42867:42867)]
2024-11-19T00:08:19,633 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/WALs/9c732c6543ff,33509,1731974869524/9c732c6543ff%2C33509%2C1731974869524.1731974869850 is not closed yet, will try archiving it next time
2024-11-19T00:08:19,633 WARN  [IPC Server handler 3 on default port 37953 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/WALs/9c732c6543ff,33509,1731974869524/9c732c6543ff%2C33509%2C1731974869524.1731974869850 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1013
2024-11-19T00:08:19,634 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/WALs/9c732c6543ff,33509,1731974869524/9c732c6543ff%2C33509%2C1731974869524.1731974869850 after 1ms
2024-11-19T00:08:19,649 DEBUG [M:0;9c732c6543ff:33509 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b2fef74a5374b3d9499d5d9130e35ea is 82, key is hbase:meta,,1/info:regioninfo/1731974870710/Put/seqid=0
2024-11-19T00:08:19,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38307 is added to blk_1073741848_1033 (size=5672)
2024-11-19T00:08:19,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44319 is added to blk_1073741848_1033 (size=5672)
2024-11-19T00:08:19,655 INFO  [M:0;9c732c6543ff:33509 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b2fef74a5374b3d9499d5d9130e35ea
2024-11-19T00:08:19,675 DEBUG [M:0;9c732c6543ff:33509 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5c92bf055c7646e1a8beafda18a4d847 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731974871139/Put/seqid=0
2024-11-19T00:08:19,679 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44319 is added to blk_1073741849_1034 (size=6118)
2024-11-19T00:08:19,680 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38307 is added to blk_1073741849_1034 (size=6118)
2024-11-19T00:08:19,680 INFO  [M:0;9c732c6543ff:33509 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5c92bf055c7646e1a8beafda18a4d847
2024-11-19T00:08:19,699 DEBUG [M:0;9c732c6543ff:33509 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2db0bf52fb2445cc8403933c5da76f1a is 69, key is 9c732c6543ff,46105,1731974869614/rs:state/1731974870079/Put/seqid=0
2024-11-19T00:08:19,703 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44319 is added to blk_1073741850_1035 (size=5156)
2024-11-19T00:08:19,703 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38307 is added to blk_1073741850_1035 (size=5156)
2024-11-19T00:08:19,704 INFO  [M:0;9c732c6543ff:33509 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2db0bf52fb2445cc8403933c5da76f1a
2024-11-19T00:08:19,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:08:19,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101085226ab0001, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:08:19,717 INFO  [RS:0;9c732c6543ff:46105 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T00:08:19,717 INFO  [RS:0;9c732c6543ff:46105 {}] regionserver.HRegionServer(1031): Exiting; stopping=9c732c6543ff,46105,1731974869614; zookeeper connection closed.
2024-11-19T00:08:19,718 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2c4a52c2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2c4a52c2
2024-11-19T00:08:19,718 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-19T00:08:19,719 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:19,720 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245 after 68056ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor192.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:19,723 DEBUG [M:0;9c732c6543ff:33509 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/712aa57af31c43f1a8e04a6eef788dbd is 52, key is load_balancer_on/state:d/1731974870758/Put/seqid=0
2024-11-19T00:08:19,727 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44319 is added to blk_1073741851_1036 (size=5056)
2024-11-19T00:08:19,728 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38307 is added to blk_1073741851_1036 (size=5056)
2024-11-19T00:08:19,728 INFO  [M:0;9c732c6543ff:33509 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/712aa57af31c43f1a8e04a6eef788dbd
2024-11-19T00:08:19,734 DEBUG [M:0;9c732c6543ff:33509 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b2fef74a5374b3d9499d5d9130e35ea as hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2b2fef74a5374b3d9499d5d9130e35ea
2024-11-19T00:08:19,739 INFO  [M:0;9c732c6543ff:33509 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2b2fef74a5374b3d9499d5d9130e35ea, entries=8, sequenceid=56, filesize=5.5 K
2024-11-19T00:08:19,739 DEBUG [M:0;9c732c6543ff:33509 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5c92bf055c7646e1a8beafda18a4d847 as hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5c92bf055c7646e1a8beafda18a4d847
2024-11-19T00:08:19,744 INFO  [M:0;9c732c6543ff:33509 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5c92bf055c7646e1a8beafda18a4d847, entries=6, sequenceid=56, filesize=6.0 K
2024-11-19T00:08:19,744 DEBUG [M:0;9c732c6543ff:33509 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2db0bf52fb2445cc8403933c5da76f1a as hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2db0bf52fb2445cc8403933c5da76f1a
2024-11-19T00:08:19,748 INFO  [M:0;9c732c6543ff:33509 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2db0bf52fb2445cc8403933c5da76f1a, entries=1, sequenceid=56, filesize=5.0 K
2024-11-19T00:08:19,749 DEBUG [M:0;9c732c6543ff:33509 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/712aa57af31c43f1a8e04a6eef788dbd as hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/712aa57af31c43f1a8e04a6eef788dbd
2024-11-19T00:08:19,754 INFO  [M:0;9c732c6543ff:33509 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/712aa57af31c43f1a8e04a6eef788dbd, entries=1, sequenceid=56, filesize=4.9 K
2024-11-19T00:08:19,755 INFO  [M:0;9c732c6543ff:33509 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 128ms, sequenceid=56, compaction requested=false
2024-11-19T00:08:19,756 INFO  [M:0;9c732c6543ff:33509 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:08:19,756 DEBUG [M:0;9c732c6543ff:33509 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731974899627Disabling compacts and flushes for region at 1731974899627Disabling writes for close at 1731974899627Obtaining lock to block concurrent updates at 1731974899627Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731974899627Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1731974899627Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731974899634 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731974899634Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731974899649 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731974899649Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731974899660 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731974899674 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731974899674Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731974899685 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731974899698 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731974899698Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731974899708 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731974899723 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731974899723Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c0a1c25: reopening flushed file at 1731974899733 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f28cfcf: reopening flushed file at 1731974899739 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f0b197d: reopening flushed file at 1731974899744 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dc62868: reopening flushed file at 1731974899749 (+5 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 128ms, sequenceid=56, compaction requested=false at 1731974899755 (+6 ms)Writing region close event to WAL at 1731974899756 (+1 ms)Closed at 1731974899756
2024-11-19T00:08:19,757 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:19,757 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:19,757 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:19,757 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:19,757 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:19,759 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38307 is added to blk_1073741847_1031 (size=757)
2024-11-19T00:08:19,759 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44319 is added to blk_1073741847_1031 (size=757)
2024-11-19T00:08:19,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:19,787 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:20,720 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:20,759 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:20,759 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:20,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:20,787 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:20,808 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:20,808 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:20,808 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:20,809 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:20,809 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:20,812 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:20,818 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:20,818 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:20,818 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:20,832 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:20,838 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:20,838 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:21,342 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T00:08:21,350 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:21,350 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:21,350 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:21,351 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:21,403 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:21,404 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:21,404 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:21,405 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:21,405 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:21,406 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:21,416 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:21,421 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:21,421 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:21,433 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:21,721 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:21,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:21,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:22,608 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1013: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-19T00:08:22,722 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:22,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:22,789 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:23,635 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/WALs/9c732c6543ff,33509,1731974869524/9c732c6543ff%2C33509%2C1731974869524.1731974869850 after 4002ms
2024-11-19T00:08:23,639 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/WALs/9c732c6543ff,33509,1731974869524/9c732c6543ff%2C33509%2C1731974869524.1731974869850 to hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/oldWALs/9c732c6543ff%2C33509%2C1731974869524.1731974869850
2024-11-19T00:08:23,644 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/MasterData/oldWALs/9c732c6543ff%2C33509%2C1731974869524.1731974869850 to hdfs://localhost:37953/user/jenkins/test-data/7e1d834b-6bfc-d41f-6b01-31138fca4aa9/oldWALs/9c732c6543ff%2C33509%2C1731974869524.1731974869850$masterlocalwal$
2024-11-19T00:08:23,644 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T00:08:23,644 INFO  [M:0;9c732c6543ff:33509 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-19T00:08:23,644 INFO  [M:0;9c732c6543ff:33509 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33509
2024-11-19T00:08:23,644 INFO  [M:0;9c732c6543ff:33509 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T00:08:23,722 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:23,747 INFO  [M:0;9c732c6543ff:33509 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T00:08:23,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:08:23,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33509-0x101085226ab0000, quorum=127.0.0.1:52353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:08:23,766 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@428b2957{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:08:23,766 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@bc30fa1{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:08:23,766 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:08:23,767 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7da73466{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:08:23,767 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5091fc79{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.log.dir/,STOPPED}
2024-11-19T00:08:23,769 WARN  [BP-1256842488-172.17.0.2-1731974868755 heartbeating to localhost/127.0.0.1:37953 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:08:23,769 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:08:23,769 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:08:23,770 WARN  [BP-1256842488-172.17.0.2-1731974868755 heartbeating to localhost/127.0.0.1:37953 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1256842488-172.17.0.2-1731974868755 (Datanode Uuid 03fef0cd-5312-4870-ab41-4af83df32bee) service to localhost/127.0.0.1:37953
2024-11-19T00:08:23,770 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/data/data3/current/BP-1256842488-172.17.0.2-1731974868755 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:08:23,771 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/data/data4/current/BP-1256842488-172.17.0.2-1731974868755 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:08:23,771 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:08:23,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:23,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:23,790 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@49d7084d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:08:23,791 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@731665df{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:08:23,791 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:08:23,791 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@570cb725{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:08:23,791 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5a382d25{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.log.dir/,STOPPED}
2024-11-19T00:08:23,793 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:08:23,793 WARN  [BP-1256842488-172.17.0.2-1731974868755 heartbeating to localhost/127.0.0.1:37953 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:08:23,793 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:08:23,793 WARN  [BP-1256842488-172.17.0.2-1731974868755 heartbeating to localhost/127.0.0.1:37953 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1256842488-172.17.0.2-1731974868755 (Datanode Uuid ea9b0a6a-d45f-4c45-aa69-040e1f5651cd) service to localhost/127.0.0.1:37953
2024-11-19T00:08:23,794 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/data/data1/current/BP-1256842488-172.17.0.2-1731974868755 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:08:23,794 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/cluster_4dbc3cb7-11d2-ac28-cf56-4a8a5ced611e/data/data2/current/BP-1256842488-172.17.0.2-1731974868755 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:08:23,794 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:08:23,807 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@16304a50{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T00:08:23,809 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e4fac25{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:08:23,809 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:08:23,809 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@347a2271{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:08:23,809 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1915705e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.log.dir/,STOPPED}
2024-11-19T00:08:23,826 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-19T00:08:23,868 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-19T00:08:23,883 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=180 (was 155)
Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:37953
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37953 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:37953
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37953
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37953
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37953 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37953
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37953 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=455 (was 436) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=215 (was 172) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4070 (was 4865)
2024-11-19T00:08:23,903 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=180, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=215, ProcessCount=11, AvailableMemoryMB=4068
2024-11-19T00:08:23,904 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-19T00:08:23,904 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.log.dir so I do NOT create it in target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1
2024-11-19T00:08:23,904 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8848d7b2-6f2b-e014-d5b8-5e87178b2b3f/hadoop.tmp.dir so I do NOT create it in target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1
2024-11-19T00:08:23,904 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/cluster_37f00d7c-f9a7-73b7-db9c-807f3eae8549, deleteOnExit=true
2024-11-19T00:08:23,904 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-19T00:08:23,904 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/test.cache.data in system properties and HBase conf
2024-11-19T00:08:23,904 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/hadoop.tmp.dir in system properties and HBase conf
2024-11-19T00:08:23,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/hadoop.log.dir in system properties and HBase conf
2024-11-19T00:08:23,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-19T00:08:23,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-19T00:08:23,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-19T00:08:23,905 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-19T00:08:23,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-19T00:08:23,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-19T00:08:23,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-19T00:08:23,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T00:08:23,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-19T00:08:23,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-19T00:08:23,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T00:08:23,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T00:08:23,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-19T00:08:23,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/nfs.dump.dir in system properties and HBase conf
2024-11-19T00:08:23,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/java.io.tmpdir in system properties and HBase conf
2024-11-19T00:08:23,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T00:08:23,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-19T00:08:23,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-19T00:08:23,926 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T00:08:24,013 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:08:24,020 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:08:24,030 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:08:24,030 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:08:24,030 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T00:08:24,031 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:08:24,031 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1dd613b3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:08:24,032 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42501282{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:08:24,186 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5fa01b00{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/java.io.tmpdir/jetty-localhost-34279-hadoop-hdfs-3_4_1-tests_jar-_-any-4006358968679578254/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T00:08:24,186 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@44b58721{HTTP/1.1, (http/1.1)}{localhost:34279}
2024-11-19T00:08:24,186 INFO  [Time-limited test {}] server.Server(415): Started @183194ms
2024-11-19T00:08:24,205 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T00:08:24,341 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:08:24,349 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:08:24,358 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:08:24,358 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:08:24,358 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T00:08:24,358 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f08894b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:08:24,359 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78617008{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:08:24,520 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3fdc15a6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/java.io.tmpdir/jetty-localhost-44369-hadoop-hdfs-3_4_1-tests_jar-_-any-2988681337759584237/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:08:24,520 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8d4c846{HTTP/1.1, (http/1.1)}{localhost:44369}
2024-11-19T00:08:24,520 INFO  [Time-limited test {}] server.Server(415): Started @183528ms
2024-11-19T00:08:24,522 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:08:24,666 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:08:24,669 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:08:24,690 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:08:24,690 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:08:24,690 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T00:08:24,694 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8afa355{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:08:24,694 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6bd1231f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:08:24,723 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:24,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:24,784 WARN  [Thread-1632 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/cluster_37f00d7c-f9a7-73b7-db9c-807f3eae8549/data/data1/current/BP-121886327-172.17.0.2-1731974903944/current, will proceed with Du for space computation calculation, 
2024-11-19T00:08:24,784 WARN  [Thread-1633 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/cluster_37f00d7c-f9a7-73b7-db9c-807f3eae8549/data/data2/current/BP-121886327-172.17.0.2-1731974903944/current, will proceed with Du for space computation calculation, 
2024-11-19T00:08:24,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:24,874 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5381625b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/java.io.tmpdir/jetty-localhost-35467-hadoop-hdfs-3_4_1-tests_jar-_-any-4110958817929325949/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:08:24,875 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22fd1e24{HTTP/1.1, (http/1.1)}{localhost:35467}
2024-11-19T00:08:24,875 INFO  [Time-limited test {}] server.Server(415): Started @183882ms
2024-11-19T00:08:24,877 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:08:24,926 WARN  [Thread-1611 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:08:24,933 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb854c72f0ba434e1 with lease ID 0x5272fd4277e7824: Processing first storage report for DS-4667f9b2-5589-4451-8e1f-5799a566edd2 from datanode DatanodeRegistration(127.0.0.1:35695, datanodeUuid=9ac7a22c-425e-4a52-afbb-4db84ed738b7, infoPort=43045, infoSecurePort=0, ipcPort=44173, storageInfo=lv=-57;cid=testClusterID;nsid=998855342;c=1731974903944)
2024-11-19T00:08:24,933 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb854c72f0ba434e1 with lease ID 0x5272fd4277e7824: from storage DS-4667f9b2-5589-4451-8e1f-5799a566edd2 node DatanodeRegistration(127.0.0.1:35695, datanodeUuid=9ac7a22c-425e-4a52-afbb-4db84ed738b7, infoPort=43045, infoSecurePort=0, ipcPort=44173, storageInfo=lv=-57;cid=testClusterID;nsid=998855342;c=1731974903944), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:08:24,933 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb854c72f0ba434e1 with lease ID 0x5272fd4277e7824: Processing first storage report for DS-e8cae619-3a8e-4fd2-88ec-3a17e3638060 from datanode DatanodeRegistration(127.0.0.1:35695, datanodeUuid=9ac7a22c-425e-4a52-afbb-4db84ed738b7, infoPort=43045, infoSecurePort=0, ipcPort=44173, storageInfo=lv=-57;cid=testClusterID;nsid=998855342;c=1731974903944)
2024-11-19T00:08:24,933 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb854c72f0ba434e1 with lease ID 0x5272fd4277e7824: from storage DS-e8cae619-3a8e-4fd2-88ec-3a17e3638060 node DatanodeRegistration(127.0.0.1:35695, datanodeUuid=9ac7a22c-425e-4a52-afbb-4db84ed738b7, infoPort=43045, infoSecurePort=0, ipcPort=44173, storageInfo=lv=-57;cid=testClusterID;nsid=998855342;c=1731974903944), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:08:25,113 WARN  [Thread-1658 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/cluster_37f00d7c-f9a7-73b7-db9c-807f3eae8549/data/data3/current/BP-121886327-172.17.0.2-1731974903944/current, will proceed with Du for space computation calculation, 
2024-11-19T00:08:25,115 WARN  [Thread-1659 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/cluster_37f00d7c-f9a7-73b7-db9c-807f3eae8549/data/data4/current/BP-121886327-172.17.0.2-1731974903944/current, will proceed with Du for space computation calculation, 
2024-11-19T00:08:25,181 WARN  [Thread-1647 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:08:25,191 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfe492dc435a3c83c with lease ID 0x5272fd4277e7825: Processing first storage report for DS-2b3090ff-357a-4dca-a4f0-f8be1ded90bb from datanode DatanodeRegistration(127.0.0.1:35521, datanodeUuid=8b61e277-8693-4d15-831b-fe55d5470dfe, infoPort=44231, infoSecurePort=0, ipcPort=39017, storageInfo=lv=-57;cid=testClusterID;nsid=998855342;c=1731974903944)
2024-11-19T00:08:25,191 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfe492dc435a3c83c with lease ID 0x5272fd4277e7825: from storage DS-2b3090ff-357a-4dca-a4f0-f8be1ded90bb node DatanodeRegistration(127.0.0.1:35521, datanodeUuid=8b61e277-8693-4d15-831b-fe55d5470dfe, infoPort=44231, infoSecurePort=0, ipcPort=39017, storageInfo=lv=-57;cid=testClusterID;nsid=998855342;c=1731974903944), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-19T00:08:25,192 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfe492dc435a3c83c with lease ID 0x5272fd4277e7825: Processing first storage report for DS-513dab39-2057-4482-8e64-669aaa3aee7c from datanode DatanodeRegistration(127.0.0.1:35521, datanodeUuid=8b61e277-8693-4d15-831b-fe55d5470dfe, infoPort=44231, infoSecurePort=0, ipcPort=39017, storageInfo=lv=-57;cid=testClusterID;nsid=998855342;c=1731974903944)
2024-11-19T00:08:25,192 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfe492dc435a3c83c with lease ID 0x5272fd4277e7825: from storage DS-513dab39-2057-4482-8e64-669aaa3aee7c node DatanodeRegistration(127.0.0.1:35521, datanodeUuid=8b61e277-8693-4d15-831b-fe55d5470dfe, infoPort=44231, infoSecurePort=0, ipcPort=39017, storageInfo=lv=-57;cid=testClusterID;nsid=998855342;c=1731974903944), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:08:25,229 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1
2024-11-19T00:08:25,241 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/cluster_37f00d7c-f9a7-73b7-db9c-807f3eae8549/zookeeper_0, clientPort=60332, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/cluster_37f00d7c-f9a7-73b7-db9c-807f3eae8549/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/cluster_37f00d7c-f9a7-73b7-db9c-807f3eae8549/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-19T00:08:25,246 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60332
2024-11-19T00:08:25,246 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:08:25,248 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:08:25,282 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741825_1001 (size=7)
2024-11-19T00:08:25,284 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741825_1001 (size=7)
2024-11-19T00:08:25,289 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885 with version=8
2024-11-19T00:08:25,289 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/hbase-staging
2024-11-19T00:08:25,292 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/9c732c6543ff:0 server-side Connection retries=45
2024-11-19T00:08:25,292 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:08:25,292 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T00:08:25,292 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T00:08:25,292 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:08:25,292 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T00:08:25,292 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-19T00:08:25,293 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T00:08:25,294 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37357
2024-11-19T00:08:25,295 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37357 connecting to ZooKeeper ensemble=127.0.0.1:60332
2024-11-19T00:08:25,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:373570x0, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T00:08:25,306 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37357-0x1010852b2400000 connected
2024-11-19T00:08:25,347 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:08:25,349 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:08:25,359 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:08:25,359 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885, hbase.cluster.distributed=false
2024-11-19T00:08:25,362 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T00:08:25,373 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37357
2024-11-19T00:08:25,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37357
2024-11-19T00:08:25,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37357
2024-11-19T00:08:25,378 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37357
2024-11-19T00:08:25,390 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37357
2024-11-19T00:08:25,414 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/9c732c6543ff:0 server-side Connection retries=45
2024-11-19T00:08:25,414 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:08:25,414 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T00:08:25,414 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T00:08:25,414 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:08:25,414 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T00:08:25,414 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-19T00:08:25,414 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T00:08:25,430 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34763
2024-11-19T00:08:25,432 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34763 connecting to ZooKeeper ensemble=127.0.0.1:60332
2024-11-19T00:08:25,433 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:08:25,435 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:08:25,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:347630x0, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T00:08:25,450 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:347630x0, quorum=127.0.0.1:60332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:08:25,450 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-19T00:08:25,461 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34763-0x1010852b2400001 connected
2024-11-19T00:08:25,469 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-19T00:08:25,470 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-19T00:08:25,475 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T00:08:25,485 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34763
2024-11-19T00:08:25,486 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34763
2024-11-19T00:08:25,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34763
2024-11-19T00:08:25,498 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34763
2024-11-19T00:08:25,498 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34763
2024-11-19T00:08:25,522 DEBUG [M:0;9c732c6543ff:37357 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9c732c6543ff:37357
2024-11-19T00:08:25,523 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9c732c6543ff,37357,1731974905292
2024-11-19T00:08:25,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:08:25,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:08:25,528 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9c732c6543ff,37357,1731974905292
2024-11-19T00:08:25,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-19T00:08:25,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:08:25,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:08:25,531 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-19T00:08:25,532 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9c732c6543ff,37357,1731974905292 from backup master directory
2024-11-19T00:08:25,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9c732c6543ff,37357,1731974905292
2024-11-19T00:08:25,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:08:25,536 WARN  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T00:08:25,536 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9c732c6543ff,37357,1731974905292
2024-11-19T00:08:25,544 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/hbase.id] with ID: dfce0295-6267-4ed0-b4bc-94f23426c824
2024-11-19T00:08:25,544 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/.tmp/hbase.id
2024-11-19T00:08:25,562 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741826_1002 (size=42)
2024-11-19T00:08:25,563 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741826_1002 (size=42)
2024-11-19T00:08:25,565 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/.tmp/hbase.id]:[hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/hbase.id]
2024-11-19T00:08:25,588 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:08:25,588 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-19T00:08:25,591 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-19T00:08:25,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:08:25,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:08:25,683 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741827_1003 (size=196)
2024-11-19T00:08:25,685 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741827_1003 (size=196)
2024-11-19T00:08:25,687 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T00:08:25,689 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-19T00:08:25,690 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:08:25,724 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:25,733 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741828_1004 (size=1189)
2024-11-19T00:08:25,733 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741828_1004 (size=1189)
2024-11-19T00:08:25,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:25,791 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:26,135 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store
2024-11-19T00:08:26,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741829_1005 (size=34)
2024-11-19T00:08:26,159 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741829_1005 (size=34)
2024-11-19T00:08:26,160 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:08:26,160 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T00:08:26,160 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:08:26,160 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:08:26,160 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T00:08:26,160 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:08:26,160 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:08:26,160 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731974906160Disabling compacts and flushes for region at 1731974906160Disabling writes for close at 1731974906160Writing region close event to WAL at 1731974906160Closed at 1731974906160
2024-11-19T00:08:26,161 WARN  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/.initializing
2024-11-19T00:08:26,161 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/WALs/9c732c6543ff,37357,1731974905292
2024-11-19T00:08:26,164 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C37357%2C1731974905292, suffix=, logDir=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/WALs/9c732c6543ff,37357,1731974905292, archiveDir=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/oldWALs, maxLogs=10
2024-11-19T00:08:26,165 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C37357%2C1731974905292.1731974906164
2024-11-19T00:08:26,177 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/WALs/9c732c6543ff,37357,1731974905292/9c732c6543ff%2C37357%2C1731974905292.1731974906164
2024-11-19T00:08:26,188 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43045:43045),(127.0.0.1/127.0.0.1:44231:44231)]
2024-11-19T00:08:26,207 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:08:26,207 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:08:26,207 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:08:26,207 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:08:26,214 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:08:26,216 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-19T00:08:26,216 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:08:26,218 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:08:26,218 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:08:26,220 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-19T00:08:26,220 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:08:26,221 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:08:26,221 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:08:26,230 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-19T00:08:26,230 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:08:26,231 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:08:26,232 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:08:26,234 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-19T00:08:26,234 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:08:26,236 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:08:26,236 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:08:26,237 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:08:26,238 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:08:26,250 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:08:26,250 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:08:26,251 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-19T00:08:26,253 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-19T00:08:26,254 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:08:26,256 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:08:26,257 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=880510, jitterRate=0.11962717771530151}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-19T00:08:26,257 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731974906207Initializing all the Stores at 1731974906210 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974906210Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974906213 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974906213Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974906213Cleaning up temporary data from old regions at 1731974906250 (+37 ms)Region opened successfully at 1731974906257 (+7 ms)
2024-11-19T00:08:26,273 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-19T00:08:26,278 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78eaa02e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0
2024-11-19T00:08:26,279 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-19T00:08:26,280 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-19T00:08:26,280 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-19T00:08:26,280 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-19T00:08:26,286 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 6 msec
2024-11-19T00:08:26,287 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-19T00:08:26,287 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-19T00:08:26,312 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-19T00:08:26,314 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-19T00:08:26,315 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-19T00:08:26,316 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-19T00:08:26,317 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-19T00:08:26,318 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-19T00:08:26,318 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-19T00:08:26,334 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-19T00:08:26,337 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-19T00:08:26,338 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-19T00:08:26,341 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-19T00:08:26,344 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-19T00:08:26,346 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-19T00:08:26,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T00:08:26,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:08:26,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T00:08:26,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:08:26,352 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9c732c6543ff,37357,1731974905292, sessionid=0x1010852b2400000, setting cluster-up flag (Was=false)
2024-11-19T00:08:26,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:08:26,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:08:26,363 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-19T00:08:26,364 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,37357,1731974905292
2024-11-19T00:08:26,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:08:26,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:08:26,375 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-19T00:08:26,377 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,37357,1731974905292
2024-11-19T00:08:26,378 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-19T00:08:26,391 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-19T00:08:26,391 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-19T00:08:26,391 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-19T00:08:26,391 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9c732c6543ff,37357,1731974905292
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-19T00:08:26,393 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:08:26,393 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:08:26,393 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:08:26,393 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:08:26,393 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9c732c6543ff:0, corePoolSize=10, maxPoolSize=10
2024-11-19T00:08:26,393 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:08:26,393 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2
2024-11-19T00:08:26,393 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:08:26,404 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:08:26,404 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-19T00:08:26,408 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:08:26,408 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-19T00:08:26,416 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731974936416
2024-11-19T00:08:26,417 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-19T00:08:26,417 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-19T00:08:26,417 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-19T00:08:26,417 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-19T00:08:26,417 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-19T00:08:26,417 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-19T00:08:26,419 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.HRegionServer(746): ClusterId : dfce0295-6267-4ed0-b4bc-94f23426c824
2024-11-19T00:08:26,419 DEBUG [RS:0;9c732c6543ff:34763 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-19T00:08:26,421 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:26,422 DEBUG [RS:0;9c732c6543ff:34763 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-19T00:08:26,422 DEBUG [RS:0;9c732c6543ff:34763 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-19T00:08:26,426 DEBUG [RS:0;9c732c6543ff:34763 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-19T00:08:26,426 DEBUG [RS:0;9c732c6543ff:34763 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11d9be6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0
2024-11-19T00:08:26,428 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-19T00:08:26,428 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-19T00:08:26,429 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-19T00:08:26,432 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-19T00:08:26,433 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-19T00:08:26,433 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974906433,5,FailOnTimeoutGroup]
2024-11-19T00:08:26,439 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974906433,5,FailOnTimeoutGroup]
2024-11-19T00:08:26,439 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:26,439 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-19T00:08:26,440 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:26,440 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:26,447 DEBUG [RS:0;9c732c6543ff:34763 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9c732c6543ff:34763
2024-11-19T00:08:26,447 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-19T00:08:26,447 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-19T00:08:26,447 DEBUG [RS:0;9c732c6543ff:34763 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-19T00:08:26,448 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.HRegionServer(2659): reportForDuty to master=9c732c6543ff,37357,1731974905292 with port=34763, startcode=1731974905413
2024-11-19T00:08:26,448 DEBUG [RS:0;9c732c6543ff:34763 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-19T00:08:26,457 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45515, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-11-19T00:08:26,458 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741831_1007 (size=1321)
2024-11-19T00:08:26,458 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741831_1007 (size=1321)
2024-11-19T00:08:26,458 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37357 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9c732c6543ff,34763,1731974905413
2024-11-19T00:08:26,458 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37357 {}] master.ServerManager(517): Registering regionserver=9c732c6543ff,34763,1731974905413
2024-11-19T00:08:26,460 DEBUG [RS:0;9c732c6543ff:34763 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885
2024-11-19T00:08:26,460 DEBUG [RS:0;9c732c6543ff:34763 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41615
2024-11-19T00:08:26,460 DEBUG [RS:0;9c732c6543ff:34763 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-19T00:08:26,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T00:08:26,463 DEBUG [RS:0;9c732c6543ff:34763 {}] zookeeper.ZKUtil(111): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9c732c6543ff,34763,1731974905413
2024-11-19T00:08:26,463 WARN  [RS:0;9c732c6543ff:34763 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T00:08:26,463 INFO  [RS:0;9c732c6543ff:34763 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:08:26,463 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-19T00:08:26,463 DEBUG [RS:0;9c732c6543ff:34763 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/WALs/9c732c6543ff,34763,1731974905413
2024-11-19T00:08:26,464 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885
2024-11-19T00:08:26,464 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9c732c6543ff,34763,1731974905413]
2024-11-19T00:08:26,470 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-19T00:08:26,482 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-19T00:08:26,483 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741832_1008 (size=32)
2024-11-19T00:08:26,484 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741832_1008 (size=32)
2024-11-19T00:08:26,485 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:08:26,489 INFO  [RS:0;9c732c6543ff:34763 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T00:08:26,489 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:26,490 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-19T00:08:26,491 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-19T00:08:26,491 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:26,492 DEBUG [RS:0;9c732c6543ff:34763 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:08:26,492 DEBUG [RS:0;9c732c6543ff:34763 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:08:26,492 DEBUG [RS:0;9c732c6543ff:34763 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:08:26,492 DEBUG [RS:0;9c732c6543ff:34763 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:08:26,492 DEBUG [RS:0;9c732c6543ff:34763 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:08:26,492 DEBUG [RS:0;9c732c6543ff:34763 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2
2024-11-19T00:08:26,492 DEBUG [RS:0;9c732c6543ff:34763 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:08:26,493 DEBUG [RS:0;9c732c6543ff:34763 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:08:26,493 DEBUG [RS:0;9c732c6543ff:34763 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:08:26,493 DEBUG [RS:0;9c732c6543ff:34763 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:08:26,493 DEBUG [RS:0;9c732c6543ff:34763 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:08:26,493 DEBUG [RS:0;9c732c6543ff:34763 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:08:26,493 DEBUG [RS:0;9c732c6543ff:34763 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3
2024-11-19T00:08:26,493 DEBUG [RS:0;9c732c6543ff:34763 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3
2024-11-19T00:08:26,496 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T00:08:26,499 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T00:08:26,499 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:08:26,499 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:08:26,499 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T00:08:26,503 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T00:08:26,503 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:08:26,504 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:08:26,504 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T00:08:26,506 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T00:08:26,507 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:08:26,510 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:08:26,511 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T00:08:26,518 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T00:08:26,518 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:08:26,522 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:08:26,523 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T00:08:26,524 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740
2024-11-19T00:08:26,525 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740
2024-11-19T00:08:26,526 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:26,526 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:26,526 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:26,526 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:26,526 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:26,526 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,34763,1731974905413-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T00:08:26,527 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T00:08:26,527 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T00:08:26,527 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T00:08:26,529 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T00:08:26,539 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:08:26,541 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=841790, jitterRate=0.07039247453212738}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T00:08:26,542 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731974906485Initializing all the Stores at 1731974906487 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974906487Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974906495 (+8 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974906495Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974906495Cleaning up temporary data from old regions at 1731974906527 (+32 ms)Region opened successfully at 1731974906542 (+15 ms)
2024-11-19T00:08:26,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T00:08:26,543 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T00:08:26,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T00:08:26,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T00:08:26,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T00:08:26,543 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T00:08:26,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731974906542Disabling compacts and flushes for region at 1731974906542Disabling writes for close at 1731974906543 (+1 ms)Writing region close event to WAL at 1731974906543Closed at 1731974906543
2024-11-19T00:08:26,545 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:08:26,546 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-19T00:08:26,546 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-19T00:08:26,552 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T00:08:26,554 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-19T00:08:26,572 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-19T00:08:26,573 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,34763,1731974905413-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:26,573 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:26,573 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.Replication(171): 9c732c6543ff,34763,1731974905413 started
2024-11-19T00:08:26,596 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:26,596 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.HRegionServer(1482): Serving as 9c732c6543ff,34763,1731974905413, RpcServer on 9c732c6543ff/172.17.0.2:34763, sessionid=0x1010852b2400001
2024-11-19T00:08:26,596 DEBUG [RS:0;9c732c6543ff:34763 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-19T00:08:26,596 DEBUG [RS:0;9c732c6543ff:34763 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9c732c6543ff,34763,1731974905413
2024-11-19T00:08:26,596 DEBUG [RS:0;9c732c6543ff:34763 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,34763,1731974905413'
2024-11-19T00:08:26,596 DEBUG [RS:0;9c732c6543ff:34763 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-19T00:08:26,599 DEBUG [RS:0;9c732c6543ff:34763 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-19T00:08:26,600 DEBUG [RS:0;9c732c6543ff:34763 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-19T00:08:26,600 DEBUG [RS:0;9c732c6543ff:34763 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-19T00:08:26,600 DEBUG [RS:0;9c732c6543ff:34763 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9c732c6543ff,34763,1731974905413
2024-11-19T00:08:26,600 DEBUG [RS:0;9c732c6543ff:34763 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,34763,1731974905413'
2024-11-19T00:08:26,600 DEBUG [RS:0;9c732c6543ff:34763 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-19T00:08:26,601 DEBUG [RS:0;9c732c6543ff:34763 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-19T00:08:26,605 DEBUG [RS:0;9c732c6543ff:34763 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-19T00:08:26,605 INFO  [RS:0;9c732c6543ff:34763 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-19T00:08:26,605 INFO  [RS:0;9c732c6543ff:34763 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-19T00:08:26,705 WARN  [9c732c6543ff:37357 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-19T00:08:26,708 INFO  [RS:0;9c732c6543ff:34763 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C34763%2C1731974905413, suffix=, logDir=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/WALs/9c732c6543ff,34763,1731974905413, archiveDir=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/oldWALs, maxLogs=32
2024-11-19T00:08:26,709 INFO  [RS:0;9c732c6543ff:34763 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C34763%2C1731974905413.1731974906709
2024-11-19T00:08:26,724 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:26,780 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:26,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:26,795 INFO  [RS:0;9c732c6543ff:34763 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/WALs/9c732c6543ff,34763,1731974905413/9c732c6543ff%2C34763%2C1731974905413.1731974906709
2024-11-19T00:08:26,805 DEBUG [RS:0;9c732c6543ff:34763 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44231:44231),(127.0.0.1/127.0.0.1:43045:43045)]
2024-11-19T00:08:26,955 DEBUG [9c732c6543ff:37357 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-19T00:08:26,956 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9c732c6543ff,34763,1731974905413
2024-11-19T00:08:26,958 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,34763,1731974905413, state=OPENING
2024-11-19T00:08:26,960 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-19T00:08:26,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:08:26,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:08:26,963 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:08:26,963 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T00:08:26,963 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:08:26,963 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,34763,1731974905413}]
2024-11-19T00:08:27,122 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-19T00:08:27,125 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37487, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-19T00:08:27,130 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-19T00:08:27,130 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:08:27,132 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C34763%2C1731974905413.meta, suffix=.meta, logDir=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/WALs/9c732c6543ff,34763,1731974905413, archiveDir=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/oldWALs, maxLogs=32
2024-11-19T00:08:27,133 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C34763%2C1731974905413.meta.1731974907133.meta
2024-11-19T00:08:27,139 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/WALs/9c732c6543ff,34763,1731974905413/9c732c6543ff%2C34763%2C1731974905413.meta.1731974907133.meta
2024-11-19T00:08:27,148 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43045:43045),(127.0.0.1/127.0.0.1:44231:44231)]
2024-11-19T00:08:27,150 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:08:27,150 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-19T00:08:27,150 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-19T00:08:27,151 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-19T00:08:27,151 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-19T00:08:27,151 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:08:27,151 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-19T00:08:27,151 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-19T00:08:27,153 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T00:08:27,154 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T00:08:27,154 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:08:27,154 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:08:27,154 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T00:08:27,155 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T00:08:27,155 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:08:27,156 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:08:27,156 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T00:08:27,156 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T00:08:27,157 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:08:27,157 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:08:27,157 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T00:08:27,158 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T00:08:27,158 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:08:27,158 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:08:27,158 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T00:08:27,159 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740
2024-11-19T00:08:27,161 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740
2024-11-19T00:08:27,162 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T00:08:27,162 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T00:08:27,163 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T00:08:27,164 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T00:08:27,165 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=877119, jitterRate=0.11531554162502289}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T00:08:27,165 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-19T00:08:27,166 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731974907151Writing region info on filesystem at 1731974907151Initializing all the Stores at 1731974907152 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974907152Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974907152Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974907152Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974907152Cleaning up temporary data from old regions at 1731974907162 (+10 ms)Running coprocessor post-open hooks at 1731974907166 (+4 ms)Region opened successfully at 1731974907166
2024-11-19T00:08:27,168 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731974907122
2024-11-19T00:08:27,170 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-19T00:08:27,170 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-19T00:08:27,171 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,34763,1731974905413
2024-11-19T00:08:27,175 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,34763,1731974905413, state=OPEN
2024-11-19T00:08:27,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T00:08:27,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T00:08:27,180 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:08:27,180 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:08:27,180 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9c732c6543ff,34763,1731974905413
2024-11-19T00:08:27,185 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-19T00:08:27,185 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,34763,1731974905413 in 217 msec
2024-11-19T00:08:27,190 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-19T00:08:27,190 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 639 msec
2024-11-19T00:08:27,192 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:08:27,192 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-19T00:08:27,198 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T00:08:27,198 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,34763,1731974905413, seqNum=-1]
2024-11-19T00:08:27,198 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T00:08:27,201 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41981, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T00:08:27,210 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 829 msec
2024-11-19T00:08:27,211 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731974907211, completionTime=-1
2024-11-19T00:08:27,211 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-19T00:08:27,211 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-19T00:08:27,214 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-19T00:08:27,214 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731974967214
2024-11-19T00:08:27,215 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731975027214
2024-11-19T00:08:27,215 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-19T00:08:27,215 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,37357,1731974905292-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:27,215 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,37357,1731974905292-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:27,227 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,37357,1731974905292-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:27,227 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9c732c6543ff:37357, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:27,228 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:27,228 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:27,230 DEBUG [master/9c732c6543ff:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-19T00:08:27,234 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.698sec
2024-11-19T00:08:27,234 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-19T00:08:27,234 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-19T00:08:27,234 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-19T00:08:27,234 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-19T00:08:27,234 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-19T00:08:27,234 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,37357,1731974905292-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T00:08:27,234 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,37357,1731974905292-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-19T00:08:27,242 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-19T00:08:27,242 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-19T00:08:27,242 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,37357,1731974905292-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:08:27,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15c1e7a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:08:27,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9c732c6543ff,37357,-1 for getting cluster id
2024-11-19T00:08:27,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-19T00:08:27,321 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dfce0295-6267-4ed0-b4bc-94f23426c824'
2024-11-19T00:08:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-19T00:08:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dfce0295-6267-4ed0-b4bc-94f23426c824"

2024-11-19T00:08:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b052b0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:08:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9c732c6543ff,37357,-1]
2024-11-19T00:08:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-19T00:08:27,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:08:27,324 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-19T00:08:27,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf4f1e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:08:27,326 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T00:08:27,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,34763,1731974905413, seqNum=-1]
2024-11-19T00:08:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T00:08:27,329 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T00:08:27,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9c732c6543ff,37357,1731974905292
2024-11-19T00:08:27,331 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:08:27,335 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-19T00:08:27,335 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-19T00:08:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 9c732c6543ff,37357,1731974905292
2024-11-19T00:08:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4442d12d
2024-11-19T00:08:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-19T00:08:27,339 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-19T00:08:27,340 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-19T00:08:27,340 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-19T00:08:27,340 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T00:08:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T00:08:27,343 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-19T00:08:27,343 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:08:27,344 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-11-19T00:08:27,345 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-19T00:08:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T00:08:27,394 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741835_1011 (size=405)
2024-11-19T00:08:27,395 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741835_1011 (size=405)
2024-11-19T00:08:27,401 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5f528629cd58cff30764836d33db81e7, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885
2024-11-19T00:08:27,415 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741836_1012 (size=88)
2024-11-19T00:08:27,421 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741836_1012 (size=88)
2024-11-19T00:08:27,424 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:08:27,424 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 5f528629cd58cff30764836d33db81e7, disabling compactions & flushes
2024-11-19T00:08:27,424 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:08:27,424 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:08:27,424 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7. after waiting 0 ms
2024-11-19T00:08:27,424 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:08:27,424 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:08:27,424 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5f528629cd58cff30764836d33db81e7:
Waiting for close lock at 1731974907424Disabling compacts and flushes for region at 1731974907424Disabling writes for close at 1731974907424Writing region close event to WAL at 1731974907424Closed at 1731974907424
2024-11-19T00:08:27,427 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-19T00:08:27,427 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731974907427"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731974907427"}]},"ts":"1731974907427"}
2024-11-19T00:08:27,431 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-19T00:08:27,432 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-19T00:08:27,433 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731974907432"}]},"ts":"1731974907432"}
2024-11-19T00:08:27,435 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-11-19T00:08:27,436 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=5f528629cd58cff30764836d33db81e7, ASSIGN}]
2024-11-19T00:08:27,438 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=5f528629cd58cff30764836d33db81e7, ASSIGN
2024-11-19T00:08:27,439 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=5f528629cd58cff30764836d33db81e7, ASSIGN; state=OFFLINE, location=9c732c6543ff,34763,1731974905413; forceNewPlan=false, retain=false
2024-11-19T00:08:27,590 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5f528629cd58cff30764836d33db81e7, regionState=OPENING, regionLocation=9c732c6543ff,34763,1731974905413
2024-11-19T00:08:27,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=5f528629cd58cff30764836d33db81e7, ASSIGN because future has completed
2024-11-19T00:08:27,593 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f528629cd58cff30764836d33db81e7, server=9c732c6543ff,34763,1731974905413}]
2024-11-19T00:08:27,725 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:27,751 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:08:27,752 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f528629cd58cff30764836d33db81e7, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:08:27,752 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 5f528629cd58cff30764836d33db81e7
2024-11-19T00:08:27,752 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:08:27,752 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5f528629cd58cff30764836d33db81e7
2024-11-19T00:08:27,752 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5f528629cd58cff30764836d33db81e7
2024-11-19T00:08:27,753 INFO  [StoreOpener-5f528629cd58cff30764836d33db81e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 5f528629cd58cff30764836d33db81e7 
2024-11-19T00:08:27,755 INFO  [StoreOpener-5f528629cd58cff30764836d33db81e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f528629cd58cff30764836d33db81e7 columnFamilyName info
2024-11-19T00:08:27,755 DEBUG [StoreOpener-5f528629cd58cff30764836d33db81e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:08:27,755 INFO  [StoreOpener-5f528629cd58cff30764836d33db81e7-1 {}] regionserver.HStore(327): Store=5f528629cd58cff30764836d33db81e7/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:08:27,755 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5f528629cd58cff30764836d33db81e7
2024-11-19T00:08:27,756 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7
2024-11-19T00:08:27,756 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7
2024-11-19T00:08:27,757 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5f528629cd58cff30764836d33db81e7
2024-11-19T00:08:27,757 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5f528629cd58cff30764836d33db81e7
2024-11-19T00:08:27,758 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5f528629cd58cff30764836d33db81e7
2024-11-19T00:08:27,761 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:08:27,762 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5f528629cd58cff30764836d33db81e7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=800881, jitterRate=0.018373191356658936}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-19T00:08:27,762 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f528629cd58cff30764836d33db81e7
2024-11-19T00:08:27,763 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5f528629cd58cff30764836d33db81e7:
Running coprocessor pre-open hook at 1731974907752Writing region info on filesystem at 1731974907752Initializing all the Stores at 1731974907753 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974907753Cleaning up temporary data from old regions at 1731974907757 (+4 ms)Running coprocessor post-open hooks at 1731974907762 (+5 ms)Region opened successfully at 1731974907762
2024-11-19T00:08:27,764 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7., pid=6, masterSystemTime=1731974907746
2024-11-19T00:08:27,766 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:08:27,766 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:08:27,768 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5f528629cd58cff30764836d33db81e7, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,34763,1731974905413
2024-11-19T00:08:27,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f528629cd58cff30764836d33db81e7, server=9c732c6543ff,34763,1731974905413 because future has completed
2024-11-19T00:08:27,779 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-19T00:08:27,779 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f528629cd58cff30764836d33db81e7, server=9c732c6543ff,34763,1731974905413 in 180 msec
2024-11-19T00:08:27,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:27,785 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-19T00:08:27,785 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=5f528629cd58cff30764836d33db81e7, ASSIGN in 343 msec
2024-11-19T00:08:27,787 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-19T00:08:27,787 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731974907787"}]},"ts":"1731974907787"}
2024-11-19T00:08:27,790 INFO  [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-11-19T00:08:27,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:27,793 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-19T00:08:27,797 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 453 msec
2024-11-19T00:08:28,726 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:28,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:28,793 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:29,726 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:29,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:29,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:30,727 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:30,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:30,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:31,728 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:31,755 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T00:08:31,756 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:31,757 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:31,757 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:31,757 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:31,758 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:31,758 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:31,782 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:31,782 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:31,782 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:31,783 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:31,783 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:31,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:31,785 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:31,789 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:31,790 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:31,790 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:31,795 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:08:31,795 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:32,470 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-19T00:08:32,471 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-11-19T00:08:32,728 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:32,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:32,795 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:33,729 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:33,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:33,796 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:34,730 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:34,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:34,796 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:35,731 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:35,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:35,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:36,253 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-19T00:08:36,253 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-19T00:08:36,254 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T00:08:36,254 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-19T00:08:36,254 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-19T00:08:36,254 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-19T00:08:36,254 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T00:08:36,255 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-11-19T00:08:36,731 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:36,787 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:36,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T00:08:37,403 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-19T00:08:37,403 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-11-19T00:08:37,407 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T00:08:37,407 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:08:37,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7., hostname=9c732c6543ff,34763,1731974905413, seqNum=2]
2024-11-19T00:08:37,421 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T00:08:37,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T00:08:37,429 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-19T00:08:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-19T00:08:37,431 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-19T00:08:37,432 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-19T00:08:37,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34763 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-11-19T00:08:37,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:08:37,596 INFO  [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 5f528629cd58cff30764836d33db81e7 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-19T00:08:37,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp/info/282f2b0a5ae94c33b6cbf0a03bc35e01 is 1080, key is row0001/info:/1731974917415/Put/seqid=0
2024-11-19T00:08:37,634 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741837_1013 (size=6033)
2024-11-19T00:08:37,634 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741837_1013 (size=6033)
2024-11-19T00:08:37,635 INFO  [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp/info/282f2b0a5ae94c33b6cbf0a03bc35e01
2024-11-19T00:08:37,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp/info/282f2b0a5ae94c33b6cbf0a03bc35e01 as hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/282f2b0a5ae94c33b6cbf0a03bc35e01
2024-11-19T00:08:37,654 INFO  [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/282f2b0a5ae94c33b6cbf0a03bc35e01, entries=1, sequenceid=5, filesize=5.9 K
2024-11-19T00:08:37,656 INFO  [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 5f528629cd58cff30764836d33db81e7 in 60ms, sequenceid=5, compaction requested=false
2024-11-19T00:08:37,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 5f528629cd58cff30764836d33db81e7:

2024-11-19T00:08:37,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:08:37,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-11-19T00:08:37,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-11-19T00:08:37,667 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-19T00:08:37,667 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 232 msec
2024-11-19T00:08:37,670 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 245 msec
2024-11-19T00:08:37,732 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:37,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:37,798 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:38,733 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:38,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:38,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:39,733 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:39,789 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:39,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:40,734 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:40,789 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:40,800 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:41,735 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:41,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:41,800 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:42,735 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:42,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:42,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:43,736 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:43,791 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:43,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:44,736 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:44,791 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:44,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:45,737 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:45,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:45,793 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088 after 68048ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor192.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:45,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:45,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta after 68045ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor192.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T00:08:46,737 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:46,793 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:46,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-19T00:08:47,443 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-19T00:08:47,446 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T00:08:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T00:08:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-19T00:08:47,448 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-19T00:08:47,449 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-19T00:08:47,449 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-19T00:08:47,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34763 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-11-19T00:08:47,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:08:47,603 INFO  [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 5f528629cd58cff30764836d33db81e7 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-19T00:08:47,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp/info/3cb1eaed626341dc9f05bd11e810e382 is 1080, key is row0002/info:/1731974927444/Put/seqid=0
2024-11-19T00:08:47,613 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741838_1014 (size=6033)
2024-11-19T00:08:47,613 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741838_1014 (size=6033)
2024-11-19T00:08:47,614 INFO  [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp/info/3cb1eaed626341dc9f05bd11e810e382
2024-11-19T00:08:47,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp/info/3cb1eaed626341dc9f05bd11e810e382 as hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/3cb1eaed626341dc9f05bd11e810e382
2024-11-19T00:08:47,624 INFO  [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/3cb1eaed626341dc9f05bd11e810e382, entries=1, sequenceid=9, filesize=5.9 K
2024-11-19T00:08:47,625 INFO  [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 5f528629cd58cff30764836d33db81e7 in 22ms, sequenceid=9, compaction requested=false
2024-11-19T00:08:47,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 5f528629cd58cff30764836d33db81e7:

2024-11-19T00:08:47,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:08:47,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-11-19T00:08:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-11-19T00:08:47,631 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-19T00:08:47,631 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 179 msec
2024-11-19T00:08:47,633 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 185 msec
2024-11-19T00:08:47,738 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:47,793 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:47,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:48,739 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:48,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:48,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:49,739 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:49,795 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:49,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:50,740 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:50,795 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:50,805 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:51,741 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:51,796 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:51,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:52,741 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:52,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:52,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:53,742 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:53,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:53,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:54,743 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:54,798 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:54,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:55,229 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-19T00:08:55,744 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:55,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:55,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:56,744 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:56,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:56,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-19T00:08:57,523 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-19T00:08:57,526 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C34763%2C1731974905413.1731974937526
2024-11-19T00:08:57,534 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:57,535 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:57,535 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:57,535 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:57,535 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:08:57,535 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/WALs/9c732c6543ff,34763,1731974905413/9c732c6543ff%2C34763%2C1731974905413.1731974906709 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/WALs/9c732c6543ff,34763,1731974905413/9c732c6543ff%2C34763%2C1731974905413.1731974937526
2024-11-19T00:08:57,537 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741833_1009 (size=5546)
2024-11-19T00:08:57,537 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44231:44231),(127.0.0.1/127.0.0.1:43045:43045)]
2024-11-19T00:08:57,537 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/WALs/9c732c6543ff,34763,1731974905413/9c732c6543ff%2C34763%2C1731974905413.1731974906709 is not closed yet, will try archiving it next time
2024-11-19T00:08:57,538 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741833_1009 (size=5546)
2024-11-19T00:08:57,541 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T00:08:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T00:08:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-19T00:08:57,545 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-19T00:08:57,546 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-19T00:08:57,546 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-19T00:08:57,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34763 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-11-19T00:08:57,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:08:57,707 INFO  [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 5f528629cd58cff30764836d33db81e7 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-19T00:08:57,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp/info/c27cc7be32bc456ba548e7ae61f3c7e5 is 1080, key is row0003/info:/1731974937525/Put/seqid=0
2024-11-19T00:08:57,728 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741840_1016 (size=6033)
2024-11-19T00:08:57,728 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741840_1016 (size=6033)
2024-11-19T00:08:57,730 INFO  [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp/info/c27cc7be32bc456ba548e7ae61f3c7e5
2024-11-19T00:08:57,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp/info/c27cc7be32bc456ba548e7ae61f3c7e5 as hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/c27cc7be32bc456ba548e7ae61f3c7e5
2024-11-19T00:08:57,745 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:57,774 INFO  [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/c27cc7be32bc456ba548e7ae61f3c7e5, entries=1, sequenceid=13, filesize=5.9 K
2024-11-19T00:08:57,776 INFO  [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 5f528629cd58cff30764836d33db81e7 in 69ms, sequenceid=13, compaction requested=true
2024-11-19T00:08:57,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 5f528629cd58cff30764836d33db81e7:

2024-11-19T00:08:57,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:08:57,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-11-19T00:08:57,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-11-19T00:08:57,781 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-19T00:08:57,781 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 233 msec
2024-11-19T00:08:57,784 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 241 msec
2024-11-19T00:08:57,800 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:57,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:58,746 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:58,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:58,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:59,746 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:59,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:08:59,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:00,747 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:00,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:00,811 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:01,747 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:01,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:01,811 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:02,748 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:02,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:02,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:03,749 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:03,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:03,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:04,749 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:04,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:04,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:05,750 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:05,805 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:05,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:06,750 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:06,805 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:06,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:07,477 INFO  [master/9c732c6543ff:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-19T00:09:07,477 INFO  [master/9c732c6543ff:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-19T00:09:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-19T00:09:07,573 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-19T00:09:07,573 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T00:09:07,576 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T00:09:07,576 DEBUG [Time-limited test {}] regionserver.HStore(1541): 5f528629cd58cff30764836d33db81e7/info is initiating minor compaction (all files)
2024-11-19T00:09:07,576 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T00:09:07,576 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:07,576 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 5f528629cd58cff30764836d33db81e7/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:09:07,576 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/282f2b0a5ae94c33b6cbf0a03bc35e01, hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/3cb1eaed626341dc9f05bd11e810e382, hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/c27cc7be32bc456ba548e7ae61f3c7e5] into tmpdir=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp, totalSize=17.7 K
2024-11-19T00:09:07,577 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 282f2b0a5ae94c33b6cbf0a03bc35e01, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731974917415
2024-11-19T00:09:07,577 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3cb1eaed626341dc9f05bd11e810e382, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1731974927444
2024-11-19T00:09:07,578 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c27cc7be32bc456ba548e7ae61f3c7e5, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731974937525
2024-11-19T00:09:07,613 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): 5f528629cd58cff30764836d33db81e7#info#compaction#46 average throughput is 3.08 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T00:09:07,614 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp/info/5fb65d52b19b42e9b1caca5d1732d467 is 1080, key is row0001/info:/1731974917415/Put/seqid=0
2024-11-19T00:09:07,651 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741841_1017 (size=8296)
2024-11-19T00:09:07,652 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741841_1017 (size=8296)
2024-11-19T00:09:07,665 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp/info/5fb65d52b19b42e9b1caca5d1732d467 as hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/5fb65d52b19b42e9b1caca5d1732d467
2024-11-19T00:09:07,676 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f528629cd58cff30764836d33db81e7/info of 5f528629cd58cff30764836d33db81e7 into 5fb65d52b19b42e9b1caca5d1732d467(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T00:09:07,676 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 5f528629cd58cff30764836d33db81e7:

2024-11-19T00:09:07,680 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C34763%2C1731974905413.1731974947679
2024-11-19T00:09:07,694 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:07,695 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:07,695 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:07,695 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:07,695 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:07,696 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/WALs/9c732c6543ff,34763,1731974905413/9c732c6543ff%2C34763%2C1731974905413.1731974937526 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/WALs/9c732c6543ff,34763,1731974905413/9c732c6543ff%2C34763%2C1731974905413.1731974947679
2024-11-19T00:09:07,700 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741839_1015 (size=2520)
2024-11-19T00:09:07,700 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741839_1015 (size=2520)
2024-11-19T00:09:07,707 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43045:43045),(127.0.0.1/127.0.0.1:44231:44231)]
2024-11-19T00:09:07,707 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/WALs/9c732c6543ff,34763,1731974905413/9c732c6543ff%2C34763%2C1731974905413.1731974937526 is not closed yet, will try archiving it next time
2024-11-19T00:09:07,708 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/WALs/9c732c6543ff,34763,1731974905413/9c732c6543ff%2C34763%2C1731974905413.1731974906709 to hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/oldWALs/9c732c6543ff%2C34763%2C1731974905413.1731974906709
2024-11-19T00:09:07,709 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T00:09:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T00:09:07,712 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-19T00:09:07,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-19T00:09:07,714 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-19T00:09:07,714 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-19T00:09:07,751 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:07,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:07,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:07,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34763 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-11-19T00:09:07,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:09:07,869 INFO  [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 5f528629cd58cff30764836d33db81e7 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-19T00:09:07,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp/info/3ab597ba9e7b4f60b53eafba823e3710 is 1080, key is row0000/info:/1731974947678/Put/seqid=0
2024-11-19T00:09:07,883 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741843_1019 (size=6033)
2024-11-19T00:09:07,883 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741843_1019 (size=6033)
2024-11-19T00:09:07,884 INFO  [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp/info/3ab597ba9e7b4f60b53eafba823e3710
2024-11-19T00:09:07,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp/info/3ab597ba9e7b4f60b53eafba823e3710 as hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/3ab597ba9e7b4f60b53eafba823e3710
2024-11-19T00:09:07,896 INFO  [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/3ab597ba9e7b4f60b53eafba823e3710, entries=1, sequenceid=18, filesize=5.9 K
2024-11-19T00:09:07,898 INFO  [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 5f528629cd58cff30764836d33db81e7 in 28ms, sequenceid=18, compaction requested=false
2024-11-19T00:09:07,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 5f528629cd58cff30764836d33db81e7:

2024-11-19T00:09:07,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:09:07,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-11-19T00:09:07,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-11-19T00:09:07,903 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-19T00:09:07,903 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 187 msec
2024-11-19T00:09:07,906 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 196 msec
2024-11-19T00:09:08,101 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/WALs/9c732c6543ff,34763,1731974905413/9c732c6543ff%2C34763%2C1731974905413.1731974937526 to hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/oldWALs/9c732c6543ff%2C34763%2C1731974905413.1731974937526
2024-11-19T00:09:08,752 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:08,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:08,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:09,752 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:09,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:09,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:10,753 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:10,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:10,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:11,753 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:11,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:11,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:12,752 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f528629cd58cff30764836d33db81e7, had cached 0 bytes from a total of 14329
2024-11-19T00:09:12,754 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:12,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:12,817 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:13,754 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:13,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:13,817 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:14,755 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:14,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:14,818 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:15,755 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:15,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:15,818 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:16,756 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:16,811 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:16,819 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:17,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-19T00:09:17,743 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-19T00:09:17,746 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C34763%2C1731974905413.1731974957746
2024-11-19T00:09:17,751 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:17,751 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:17,751 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:17,751 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:17,752 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:17,752 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/WALs/9c732c6543ff,34763,1731974905413/9c732c6543ff%2C34763%2C1731974905413.1731974947679 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/WALs/9c732c6543ff,34763,1731974905413/9c732c6543ff%2C34763%2C1731974905413.1731974957746
2024-11-19T00:09:17,752 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44231:44231),(127.0.0.1/127.0.0.1:43045:43045)]
2024-11-19T00:09:17,753 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/WALs/9c732c6543ff,34763,1731974905413/9c732c6543ff%2C34763%2C1731974905413.1731974947679 is not closed yet, will try archiving it next time
2024-11-19T00:09:17,753 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-19T00:09:17,753 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T00:09:17,753 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:09:17,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:09:17,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:09:17,753 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-19T00:09:17,753 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741842_1018 (size=2026)
2024-11-19T00:09:17,753 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=345624216, stopped=false
2024-11-19T00:09:17,753 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9c732c6543ff,37357,1731974905292
2024-11-19T00:09:17,753 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-19T00:09:17,754 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741842_1018 (size=2026)
2024-11-19T00:09:17,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T00:09:17,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T00:09:17,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:09:17,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:09:17,755 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T00:09:17,755 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T00:09:17,756 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:09:17,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:09:17,756 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9c732c6543ff,34763,1731974905413' *****
2024-11-19T00:09:17,756 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-19T00:09:17,756 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-19T00:09:17,756 INFO  [RS:0;9c732c6543ff:34763 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-19T00:09:17,756 INFO  [RS:0;9c732c6543ff:34763 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-19T00:09:17,756 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:09:17,756 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.HRegionServer(3091): Received CLOSE for 5f528629cd58cff30764836d33db81e7
2024-11-19T00:09:17,756 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:09:17,756 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:17,757 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.HRegionServer(959): stopping server 9c732c6543ff,34763,1731974905413
2024-11-19T00:09:17,757 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T00:09:17,757 INFO  [RS:0;9c732c6543ff:34763 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9c732c6543ff:34763.
2024-11-19T00:09:17,757 DEBUG [RS:0;9c732c6543ff:34763 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:09:17,757 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5f528629cd58cff30764836d33db81e7, disabling compactions & flushes
2024-11-19T00:09:17,757 DEBUG [RS:0;9c732c6543ff:34763 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:09:17,757 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:09:17,757 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:09:17,757 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-19T00:09:17,757 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7. after waiting 0 ms
2024-11-19T00:09:17,757 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-19T00:09:17,757 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:09:17,757 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-19T00:09:17,757 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-19T00:09:17,757 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-19T00:09:17,757 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5f528629cd58cff30764836d33db81e7 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-19T00:09:17,757 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-19T00:09:17,757 DEBUG [RS:0;9c732c6543ff:34763 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 5f528629cd58cff30764836d33db81e7=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.}
2024-11-19T00:09:17,757 DEBUG [RS:0;9c732c6543ff:34763 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5f528629cd58cff30764836d33db81e7
2024-11-19T00:09:17,757 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T00:09:17,757 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T00:09:17,757 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T00:09:17,757 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T00:09:17,757 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T00:09:17,758 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-11-19T00:09:17,761 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp/info/d65939a3912f415b8684762f5b1f91c3 is 1080, key is row0001/info:/1731974957744/Put/seqid=0
2024-11-19T00:09:17,766 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741845_1021 (size=6033)
2024-11-19T00:09:17,766 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741845_1021 (size=6033)
2024-11-19T00:09:17,766 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp/info/d65939a3912f415b8684762f5b1f91c3
2024-11-19T00:09:17,772 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/.tmp/info/d65939a3912f415b8684762f5b1f91c3 as hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/d65939a3912f415b8684762f5b1f91c3
2024-11-19T00:09:17,775 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740/.tmp/info/dac9cef8a4904abaaa0a701973c8323d is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7./info:regioninfo/1731974907767/Put/seqid=0
2024-11-19T00:09:17,777 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/d65939a3912f415b8684762f5b1f91c3, entries=1, sequenceid=22, filesize=5.9 K
2024-11-19T00:09:17,778 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 5f528629cd58cff30764836d33db81e7 in 21ms, sequenceid=22, compaction requested=true
2024-11-19T00:09:17,780 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/282f2b0a5ae94c33b6cbf0a03bc35e01, hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/3cb1eaed626341dc9f05bd11e810e382, hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/c27cc7be32bc456ba548e7ae61f3c7e5] to archive
2024-11-19T00:09:17,781 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-19T00:09:17,782 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741846_1022 (size=7308)
2024-11-19T00:09:17,782 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741846_1022 (size=7308)
2024-11-19T00:09:17,782 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740/.tmp/info/dac9cef8a4904abaaa0a701973c8323d
2024-11-19T00:09:17,783 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/282f2b0a5ae94c33b6cbf0a03bc35e01 to hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/282f2b0a5ae94c33b6cbf0a03bc35e01
2024-11-19T00:09:17,785 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/3cb1eaed626341dc9f05bd11e810e382 to hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/3cb1eaed626341dc9f05bd11e810e382
2024-11-19T00:09:17,786 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/c27cc7be32bc456ba548e7ae61f3c7e5 to hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/info/c27cc7be32bc456ba548e7ae61f3c7e5
2024-11-19T00:09:17,786 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=9c732c6543ff:37357 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-19T00:09:17,786 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [282f2b0a5ae94c33b6cbf0a03bc35e01=6033, 3cb1eaed626341dc9f05bd11e810e382=6033, c27cc7be32bc456ba548e7ae61f3c7e5=6033]
2024-11-19T00:09:17,790 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5f528629cd58cff30764836d33db81e7/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-11-19T00:09:17,790 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:09:17,790 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5f528629cd58cff30764836d33db81e7:
Waiting for close lock at 1731974957757Running coprocessor pre-close hooks at 1731974957757Disabling compacts and flushes for region at 1731974957757Disabling writes for close at 1731974957757Obtaining lock to block concurrent updates at 1731974957757Preparing flush snapshotting stores in 5f528629cd58cff30764836d33db81e7 at 1731974957757Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1731974957757Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7. at 1731974957758 (+1 ms)Flushing 5f528629cd58cff30764836d33db81e7/info: creating writer at 1731974957758Flushing 5f528629cd58cff30764836d33db81e7/info: appending metadata at 1731974957761 (+3 ms)Flushing 5f528629cd58cff30764836d33db81e7/info: closing flushed file at 1731974957761Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ab03ffb: reopening flushed file at 1731974957772 (+11 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 5f528629cd58cff30764836d33db81e7 in 21ms, sequenceid=22, compaction requested=true at 1731974957778 (+6 ms)Writing region close event to WAL at 1731974957787 (+9 ms)Running coprocessor post-close hooks at 1731974957790 (+3 ms)Closed at 1731974957790
2024-11-19T00:09:17,790 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731974907339.5f528629cd58cff30764836d33db81e7.
2024-11-19T00:09:17,801 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740/.tmp/ns/e323973e42e94b298d28a283c492d4ba is 43, key is default/ns:d/1731974907201/Put/seqid=0
2024-11-19T00:09:17,805 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741847_1023 (size=5153)
2024-11-19T00:09:17,806 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741847_1023 (size=5153)
2024-11-19T00:09:17,806 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740/.tmp/ns/e323973e42e94b298d28a283c492d4ba
2024-11-19T00:09:17,811 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:17,819 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:17,825 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740/.tmp/table/602ef2c348ff4fd999cc80218f14bfb6 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1731974907787/Put/seqid=0
2024-11-19T00:09:17,829 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741848_1024 (size=5508)
2024-11-19T00:09:17,829 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741848_1024 (size=5508)
2024-11-19T00:09:17,829 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740/.tmp/table/602ef2c348ff4fd999cc80218f14bfb6
2024-11-19T00:09:17,834 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740/.tmp/info/dac9cef8a4904abaaa0a701973c8323d as hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740/info/dac9cef8a4904abaaa0a701973c8323d
2024-11-19T00:09:17,838 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740/info/dac9cef8a4904abaaa0a701973c8323d, entries=10, sequenceid=11, filesize=7.1 K
2024-11-19T00:09:17,839 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740/.tmp/ns/e323973e42e94b298d28a283c492d4ba as hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740/ns/e323973e42e94b298d28a283c492d4ba
2024-11-19T00:09:17,843 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740/ns/e323973e42e94b298d28a283c492d4ba, entries=2, sequenceid=11, filesize=5.0 K
2024-11-19T00:09:17,844 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740/.tmp/table/602ef2c348ff4fd999cc80218f14bfb6 as hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740/table/602ef2c348ff4fd999cc80218f14bfb6
2024-11-19T00:09:17,848 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740/table/602ef2c348ff4fd999cc80218f14bfb6, entries=2, sequenceid=11, filesize=5.4 K
2024-11-19T00:09:17,849 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 92ms, sequenceid=11, compaction requested=false
2024-11-19T00:09:17,853 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-19T00:09:17,854 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T00:09:17,854 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T00:09:17,854 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731974957757Running coprocessor pre-close hooks at 1731974957757Disabling compacts and flushes for region at 1731974957757Disabling writes for close at 1731974957757Obtaining lock to block concurrent updates at 1731974957758 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731974957758Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1731974957758Flushing stores of hbase:meta,,1.1588230740 at 1731974957758Flushing 1588230740/info: creating writer at 1731974957759 (+1 ms)Flushing 1588230740/info: appending metadata at 1731974957774 (+15 ms)Flushing 1588230740/info: closing flushed file at 1731974957775 (+1 ms)Flushing 1588230740/ns: creating writer at 1731974957787 (+12 ms)Flushing 1588230740/ns: appending metadata at 1731974957801 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731974957801Flushing 1588230740/table: creating writer at 1731974957810 (+9 ms)Flushing 1588230740/table: appending metadata at 1731974957824 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731974957824Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@737149df: reopening flushed file at 1731974957833 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2556f7cb: reopening flushed file at 1731974957838 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10247e82: reopening flushed file at 1731974957843 (+5 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 92ms, sequenceid=11, compaction requested=false at 1731974957849 (+6 ms)Writing region close event to WAL at 1731974957850 (+1 ms)Running coprocessor post-close hooks at 1731974957854 (+4 ms)Closed at 1731974957854
2024-11-19T00:09:17,854 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-19T00:09:17,957 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.HRegionServer(976): stopping server 9c732c6543ff,34763,1731974905413; all regions closed.
2024-11-19T00:09:17,958 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:17,958 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:17,958 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:17,958 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:17,958 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:17,960 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741834_1010 (size=3306)
2024-11-19T00:09:17,961 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741834_1010 (size=3306)
2024-11-19T00:09:17,963 DEBUG [RS:0;9c732c6543ff:34763 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/oldWALs
2024-11-19T00:09:17,963 INFO  [RS:0;9c732c6543ff:34763 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9c732c6543ff%2C34763%2C1731974905413.meta:.meta(num 1731974907133)
2024-11-19T00:09:17,963 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:17,963 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:17,963 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:17,964 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:17,964 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:17,965 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741844_1020 (size=1252)
2024-11-19T00:09:17,966 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741844_1020 (size=1252)
2024-11-19T00:09:17,968 DEBUG [RS:0;9c732c6543ff:34763 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/oldWALs
2024-11-19T00:09:17,968 INFO  [RS:0;9c732c6543ff:34763 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9c732c6543ff%2C34763%2C1731974905413:(num 1731974957746)
2024-11-19T00:09:17,968 DEBUG [RS:0;9c732c6543ff:34763 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:09:17,968 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T00:09:17,968 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T00:09:17,969 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.ChoreService(370): Chore service for: regionserver/9c732c6543ff:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-19T00:09:17,969 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T00:09:17,969 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T00:09:17,969 INFO  [RS:0;9c732c6543ff:34763 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34763
2024-11-19T00:09:17,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9c732c6543ff,34763,1731974905413
2024-11-19T00:09:17,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T00:09:17,971 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T00:09:17,973 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9c732c6543ff,34763,1731974905413]
2024-11-19T00:09:17,974 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9c732c6543ff,34763,1731974905413 already deleted, retry=false
2024-11-19T00:09:17,974 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9c732c6543ff,34763,1731974905413 expired; onlineServers=0
2024-11-19T00:09:17,974 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9c732c6543ff,37357,1731974905292' *****
2024-11-19T00:09:17,974 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-19T00:09:17,974 INFO  [M:0;9c732c6543ff:37357 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T00:09:17,974 INFO  [M:0;9c732c6543ff:37357 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T00:09:17,974 DEBUG [M:0;9c732c6543ff:37357 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-19T00:09:17,974 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-19T00:09:17,974 DEBUG [M:0;9c732c6543ff:37357 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-19T00:09:17,974 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974906433 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974906433,5,FailOnTimeoutGroup]
2024-11-19T00:09:17,974 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974906433 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974906433,5,FailOnTimeoutGroup]
2024-11-19T00:09:17,974 INFO  [M:0;9c732c6543ff:37357 {}] hbase.ChoreService(370): Chore service for: master/9c732c6543ff:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-19T00:09:17,975 INFO  [M:0;9c732c6543ff:37357 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T00:09:17,975 DEBUG [M:0;9c732c6543ff:37357 {}] master.HMaster(1795): Stopping service threads
2024-11-19T00:09:17,975 INFO  [M:0;9c732c6543ff:37357 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-19T00:09:17,975 INFO  [M:0;9c732c6543ff:37357 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T00:09:17,975 INFO  [M:0;9c732c6543ff:37357 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-19T00:09:17,975 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-19T00:09:17,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-19T00:09:17,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:09:17,976 DEBUG [M:0;9c732c6543ff:37357 {}] zookeeper.ZKUtil(347): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-19T00:09:17,976 WARN  [M:0;9c732c6543ff:37357 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-19T00:09:17,977 INFO  [M:0;9c732c6543ff:37357 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/.lastflushedseqids
2024-11-19T00:09:17,982 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741849_1025 (size=130)
2024-11-19T00:09:17,982 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741849_1025 (size=130)
2024-11-19T00:09:17,982 INFO  [M:0;9c732c6543ff:37357 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-19T00:09:17,982 INFO  [M:0;9c732c6543ff:37357 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-19T00:09:17,983 DEBUG [M:0;9c732c6543ff:37357 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T00:09:17,983 INFO  [M:0;9c732c6543ff:37357 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:09:17,983 DEBUG [M:0;9c732c6543ff:37357 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:09:17,983 DEBUG [M:0;9c732c6543ff:37357 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T00:09:17,983 DEBUG [M:0;9c732c6543ff:37357 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:09:17,983 INFO  [M:0;9c732c6543ff:37357 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.61 KB heapSize=55.02 KB
2024-11-19T00:09:17,998 DEBUG [M:0;9c732c6543ff:37357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/97b46ba1d1584b45958943c027f818dd is 82, key is hbase:meta,,1/info:regioninfo/1731974907171/Put/seqid=0
2024-11-19T00:09:18,003 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741850_1026 (size=5672)
2024-11-19T00:09:18,003 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741850_1026 (size=5672)
2024-11-19T00:09:18,003 INFO  [M:0;9c732c6543ff:37357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/97b46ba1d1584b45958943c027f818dd
2024-11-19T00:09:18,023 DEBUG [M:0;9c732c6543ff:37357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ae079b81dff246a1bef54f1465b40bd3 is 799, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731974907796/Put/seqid=0
2024-11-19T00:09:18,027 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741851_1027 (size=7825)
2024-11-19T00:09:18,027 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741851_1027 (size=7825)
2024-11-19T00:09:18,027 INFO  [M:0;9c732c6543ff:37357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.01 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ae079b81dff246a1bef54f1465b40bd3
2024-11-19T00:09:18,032 INFO  [M:0;9c732c6543ff:37357 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ae079b81dff246a1bef54f1465b40bd3
2024-11-19T00:09:18,046 DEBUG [M:0;9c732c6543ff:37357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/975d74c9878d451b81437ed74ba8b674 is 69, key is 9c732c6543ff,34763,1731974905413/rs:state/1731974906458/Put/seqid=0
2024-11-19T00:09:18,051 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741852_1028 (size=5156)
2024-11-19T00:09:18,051 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741852_1028 (size=5156)
2024-11-19T00:09:18,052 INFO  [M:0;9c732c6543ff:37357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/975d74c9878d451b81437ed74ba8b674
2024-11-19T00:09:18,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:09:18,073 INFO  [RS:0;9c732c6543ff:34763 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T00:09:18,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34763-0x1010852b2400001, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:09:18,073 INFO  [RS:0;9c732c6543ff:34763 {}] regionserver.HRegionServer(1031): Exiting; stopping=9c732c6543ff,34763,1731974905413; zookeeper connection closed.
2024-11-19T00:09:18,073 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@63e69b4b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@63e69b4b
2024-11-19T00:09:18,073 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-19T00:09:18,074 DEBUG [M:0;9c732c6543ff:37357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c251de05ca4d4b4397bd134c845e4790 is 52, key is load_balancer_on/state:d/1731974907333/Put/seqid=0
2024-11-19T00:09:18,082 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741853_1029 (size=5056)
2024-11-19T00:09:18,082 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741853_1029 (size=5056)
2024-11-19T00:09:18,083 INFO  [M:0;9c732c6543ff:37357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c251de05ca4d4b4397bd134c845e4790
2024-11-19T00:09:18,087 DEBUG [M:0;9c732c6543ff:37357 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/97b46ba1d1584b45958943c027f818dd as hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/97b46ba1d1584b45958943c027f818dd
2024-11-19T00:09:18,091 INFO  [M:0;9c732c6543ff:37357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/97b46ba1d1584b45958943c027f818dd, entries=8, sequenceid=121, filesize=5.5 K
2024-11-19T00:09:18,092 DEBUG [M:0;9c732c6543ff:37357 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ae079b81dff246a1bef54f1465b40bd3 as hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ae079b81dff246a1bef54f1465b40bd3
2024-11-19T00:09:18,096 INFO  [M:0;9c732c6543ff:37357 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ae079b81dff246a1bef54f1465b40bd3
2024-11-19T00:09:18,096 INFO  [M:0;9c732c6543ff:37357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ae079b81dff246a1bef54f1465b40bd3, entries=14, sequenceid=121, filesize=7.6 K
2024-11-19T00:09:18,097 DEBUG [M:0;9c732c6543ff:37357 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/975d74c9878d451b81437ed74ba8b674 as hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/975d74c9878d451b81437ed74ba8b674
2024-11-19T00:09:18,101 INFO  [M:0;9c732c6543ff:37357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/975d74c9878d451b81437ed74ba8b674, entries=1, sequenceid=121, filesize=5.0 K
2024-11-19T00:09:18,102 DEBUG [M:0;9c732c6543ff:37357 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c251de05ca4d4b4397bd134c845e4790 as hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c251de05ca4d4b4397bd134c845e4790
2024-11-19T00:09:18,106 INFO  [M:0;9c732c6543ff:37357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41615/user/jenkins/test-data/358474a4-92ae-5010-7212-c770cdc26885/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c251de05ca4d4b4397bd134c845e4790, entries=1, sequenceid=121, filesize=4.9 K
2024-11-19T00:09:18,107 INFO  [M:0;9c732c6543ff:37357 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.61 KB/44659, heapSize ~54.95 KB/56272, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 124ms, sequenceid=121, compaction requested=false
2024-11-19T00:09:18,108 INFO  [M:0;9c732c6543ff:37357 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:09:18,109 DEBUG [M:0;9c732c6543ff:37357 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731974957982Disabling compacts and flushes for region at 1731974957982Disabling writes for close at 1731974957983 (+1 ms)Obtaining lock to block concurrent updates at 1731974957983Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731974957983Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44659, getHeapSize=56272, getOffHeapSize=0, getCellsCount=140 at 1731974957983Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731974957984 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731974957984Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731974957998 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731974957998Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731974958008 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731974958022 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731974958022Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731974958032 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731974958046 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731974958046Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731974958057 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731974958073 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731974958073Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7457bf39: reopening flushed file at 1731974958087 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61e59066: reopening flushed file at 1731974958091 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79693420: reopening flushed file at 1731974958096 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e47d9b6: reopening flushed file at 1731974958101 (+5 ms)Finished flush of dataSize ~43.61 KB/44659, heapSize ~54.95 KB/56272, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 124ms, sequenceid=121, compaction requested=false at 1731974958107 (+6 ms)Writing region close event to WAL at 1731974958108 (+1 ms)Closed at 1731974958108
2024-11-19T00:09:18,109 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:18,109 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:18,109 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:18,109 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:18,109 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:09:18,111 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35521 is added to blk_1073741830_1006 (size=53056)
2024-11-19T00:09:18,112 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741830_1006 (size=53056)
2024-11-19T00:09:18,112 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T00:09:18,112 INFO  [M:0;9c732c6543ff:37357 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-19T00:09:18,112 INFO  [M:0;9c732c6543ff:37357 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37357
2024-11-19T00:09:18,112 INFO  [M:0;9c732c6543ff:37357 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T00:09:18,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:09:18,214 INFO  [M:0;9c732c6543ff:37357 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T00:09:18,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37357-0x1010852b2400000, quorum=127.0.0.1:60332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:09:18,217 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5381625b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:09:18,217 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22fd1e24{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:09:18,217 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:09:18,217 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6bd1231f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:09:18,217 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8afa355{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/hadoop.log.dir/,STOPPED}
2024-11-19T00:09:18,219 WARN  [BP-121886327-172.17.0.2-1731974903944 heartbeating to localhost/127.0.0.1:41615 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:09:18,219 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:09:18,219 WARN  [BP-121886327-172.17.0.2-1731974903944 heartbeating to localhost/127.0.0.1:41615 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-121886327-172.17.0.2-1731974903944 (Datanode Uuid 8b61e277-8693-4d15-831b-fe55d5470dfe) service to localhost/127.0.0.1:41615
2024-11-19T00:09:18,219 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:09:18,219 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/cluster_37f00d7c-f9a7-73b7-db9c-807f3eae8549/data/data3/current/BP-121886327-172.17.0.2-1731974903944 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:09:18,220 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/cluster_37f00d7c-f9a7-73b7-db9c-807f3eae8549/data/data4/current/BP-121886327-172.17.0.2-1731974903944 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:09:18,220 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:09:18,222 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3fdc15a6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:09:18,222 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8d4c846{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:09:18,222 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:09:18,222 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78617008{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:09:18,222 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f08894b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/hadoop.log.dir/,STOPPED}
2024-11-19T00:09:18,224 WARN  [BP-121886327-172.17.0.2-1731974903944 heartbeating to localhost/127.0.0.1:41615 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:09:18,224 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:09:18,224 WARN  [BP-121886327-172.17.0.2-1731974903944 heartbeating to localhost/127.0.0.1:41615 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-121886327-172.17.0.2-1731974903944 (Datanode Uuid 9ac7a22c-425e-4a52-afbb-4db84ed738b7) service to localhost/127.0.0.1:41615
2024-11-19T00:09:18,224 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:09:18,224 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/cluster_37f00d7c-f9a7-73b7-db9c-807f3eae8549/data/data1/current/BP-121886327-172.17.0.2-1731974903944 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:09:18,224 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/cluster_37f00d7c-f9a7-73b7-db9c-807f3eae8549/data/data2/current/BP-121886327-172.17.0.2-1731974903944 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:09:18,225 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:09:18,230 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5fa01b00{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T00:09:18,231 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@44b58721{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:09:18,231 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:09:18,231 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42501282{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:09:18,231 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1dd613b3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/hadoop.log.dir/,STOPPED}
2024-11-19T00:09:18,237 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-19T00:09:18,254 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-19T00:09:18,262 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=206 (was 180)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41615
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41615
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41615
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41615 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:41615
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41615 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41615 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:41615
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=483 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=270 (was 215) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3660 (was 4068)
2024-11-19T00:09:18,270 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=206, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=270, ProcessCount=11, AvailableMemoryMB=3661
2024-11-19T00:09:18,270 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-19T00:09:18,271 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/hadoop.log.dir so I do NOT create it in target/test-data/417ac65e-d831-562f-c917-198a2eb11073
2024-11-19T00:09:18,271 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2ed1897a-ffbf-c0c0-a678-affc3c06d5e1/hadoop.tmp.dir so I do NOT create it in target/test-data/417ac65e-d831-562f-c917-198a2eb11073
2024-11-19T00:09:18,271 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/cluster_4aa9ef7b-a731-47c8-baac-55880e8d45de, deleteOnExit=true
2024-11-19T00:09:18,271 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-19T00:09:18,271 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/test.cache.data in system properties and HBase conf
2024-11-19T00:09:18,271 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/hadoop.tmp.dir in system properties and HBase conf
2024-11-19T00:09:18,271 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/hadoop.log.dir in system properties and HBase conf
2024-11-19T00:09:18,271 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-19T00:09:18,271 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-19T00:09:18,271 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-19T00:09:18,271 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-19T00:09:18,271 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-19T00:09:18,272 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-19T00:09:18,272 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-19T00:09:18,272 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T00:09:18,272 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-19T00:09:18,272 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-19T00:09:18,272 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T00:09:18,272 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T00:09:18,272 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-19T00:09:18,272 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/nfs.dump.dir in system properties and HBase conf
2024-11-19T00:09:18,272 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/java.io.tmpdir in system properties and HBase conf
2024-11-19T00:09:18,272 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T00:09:18,272 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-19T00:09:18,272 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-19T00:09:18,285 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T00:09:18,340 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:09:18,344 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:09:18,345 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:09:18,345 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:09:18,345 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T00:09:18,346 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:09:18,346 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@540b6b2e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:09:18,346 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51e0317a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:09:18,460 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@e4041f6{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/java.io.tmpdir/jetty-localhost-35533-hadoop-hdfs-3_4_1-tests_jar-_-any-15391449953263559969/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T00:09:18,461 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@c5b7d7a{HTTP/1.1, (http/1.1)}{localhost:35533}
2024-11-19T00:09:18,461 INFO  [Time-limited test {}] server.Server(415): Started @237468ms
2024-11-19T00:09:18,474 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T00:09:18,538 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:09:18,539 INFO  [regionserver/9c732c6543ff:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T00:09:18,541 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:09:18,542 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:09:18,542 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:09:18,542 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T00:09:18,542 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75d9b484{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:09:18,543 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@516e643a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:09:18,657 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7305dd28{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/java.io.tmpdir/jetty-localhost-45641-hadoop-hdfs-3_4_1-tests_jar-_-any-5542168240190204596/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:09:18,657 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f8818bb{HTTP/1.1, (http/1.1)}{localhost:45641}
2024-11-19T00:09:18,657 INFO  [Time-limited test {}] server.Server(415): Started @237665ms
2024-11-19T00:09:18,659 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:09:18,688 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:09:18,690 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:09:18,691 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:09:18,691 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:09:18,691 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T00:09:18,692 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@412b5320{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:09:18,692 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3868302b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:09:18,739 WARN  [Thread-1946 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/cluster_4aa9ef7b-a731-47c8-baac-55880e8d45de/data/data1/current/BP-344767392-172.17.0.2-1731974958292/current, will proceed with Du for space computation calculation, 
2024-11-19T00:09:18,739 WARN  [Thread-1947 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/cluster_4aa9ef7b-a731-47c8-baac-55880e8d45de/data/data2/current/BP-344767392-172.17.0.2-1731974958292/current, will proceed with Du for space computation calculation, 
2024-11-19T00:09:18,755 WARN  [Thread-1925 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:09:18,757 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:18,758 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x71dbca1d9c7d3d32 with lease ID 0x7b92ccbd8713b481: Processing first storage report for DS-50c7afb6-c889-41d8-b841-8d87bfbcb0b9 from datanode DatanodeRegistration(127.0.0.1:33047, datanodeUuid=439bf844-b551-4b29-ae83-aa33a47a0a86, infoPort=36629, infoSecurePort=0, ipcPort=46447, storageInfo=lv=-57;cid=testClusterID;nsid=109867591;c=1731974958292)
2024-11-19T00:09:18,758 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x71dbca1d9c7d3d32 with lease ID 0x7b92ccbd8713b481: from storage DS-50c7afb6-c889-41d8-b841-8d87bfbcb0b9 node DatanodeRegistration(127.0.0.1:33047, datanodeUuid=439bf844-b551-4b29-ae83-aa33a47a0a86, infoPort=36629, infoSecurePort=0, ipcPort=46447, storageInfo=lv=-57;cid=testClusterID;nsid=109867591;c=1731974958292), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-19T00:09:18,758 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x71dbca1d9c7d3d32 with lease ID 0x7b92ccbd8713b481: Processing first storage report for DS-43faa004-2748-4419-9f44-5dc754370f21 from datanode DatanodeRegistration(127.0.0.1:33047, datanodeUuid=439bf844-b551-4b29-ae83-aa33a47a0a86, infoPort=36629, infoSecurePort=0, ipcPort=46447, storageInfo=lv=-57;cid=testClusterID;nsid=109867591;c=1731974958292)
2024-11-19T00:09:18,758 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x71dbca1d9c7d3d32 with lease ID 0x7b92ccbd8713b481: from storage DS-43faa004-2748-4419-9f44-5dc754370f21 node DatanodeRegistration(127.0.0.1:33047, datanodeUuid=439bf844-b551-4b29-ae83-aa33a47a0a86, infoPort=36629, infoSecurePort=0, ipcPort=46447, storageInfo=lv=-57;cid=testClusterID;nsid=109867591;c=1731974958292), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:09:18,806 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@719b1e37{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/java.io.tmpdir/jetty-localhost-39393-hadoop-hdfs-3_4_1-tests_jar-_-any-18390290763940024006/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:09:18,806 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34accf12{HTTP/1.1, (http/1.1)}{localhost:39393}
2024-11-19T00:09:18,806 INFO  [Time-limited test {}] server.Server(415): Started @237814ms
2024-11-19T00:09:18,807 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:09:18,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:18,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:18,881 WARN  [Thread-1972 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/cluster_4aa9ef7b-a731-47c8-baac-55880e8d45de/data/data3/current/BP-344767392-172.17.0.2-1731974958292/current, will proceed with Du for space computation calculation, 
2024-11-19T00:09:18,881 WARN  [Thread-1973 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/cluster_4aa9ef7b-a731-47c8-baac-55880e8d45de/data/data4/current/BP-344767392-172.17.0.2-1731974958292/current, will proceed with Du for space computation calculation, 
2024-11-19T00:09:18,896 WARN  [Thread-1961 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:09:18,899 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x61d5ad9a55749597 with lease ID 0x7b92ccbd8713b482: Processing first storage report for DS-550d7403-482b-4387-8334-49b8812ac73e from datanode DatanodeRegistration(127.0.0.1:39605, datanodeUuid=40c6372d-cffd-48b8-93fc-1738c3405346, infoPort=42451, infoSecurePort=0, ipcPort=45119, storageInfo=lv=-57;cid=testClusterID;nsid=109867591;c=1731974958292)
2024-11-19T00:09:18,899 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x61d5ad9a55749597 with lease ID 0x7b92ccbd8713b482: from storage DS-550d7403-482b-4387-8334-49b8812ac73e node DatanodeRegistration(127.0.0.1:39605, datanodeUuid=40c6372d-cffd-48b8-93fc-1738c3405346, infoPort=42451, infoSecurePort=0, ipcPort=45119, storageInfo=lv=-57;cid=testClusterID;nsid=109867591;c=1731974958292), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:09:18,899 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x61d5ad9a55749597 with lease ID 0x7b92ccbd8713b482: Processing first storage report for DS-775f076f-a614-47ba-a464-0855030cafe9 from datanode DatanodeRegistration(127.0.0.1:39605, datanodeUuid=40c6372d-cffd-48b8-93fc-1738c3405346, infoPort=42451, infoSecurePort=0, ipcPort=45119, storageInfo=lv=-57;cid=testClusterID;nsid=109867591;c=1731974958292)
2024-11-19T00:09:18,899 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x61d5ad9a55749597 with lease ID 0x7b92ccbd8713b482: from storage DS-775f076f-a614-47ba-a464-0855030cafe9 node DatanodeRegistration(127.0.0.1:39605, datanodeUuid=40c6372d-cffd-48b8-93fc-1738c3405346, infoPort=42451, infoSecurePort=0, ipcPort=45119, storageInfo=lv=-57;cid=testClusterID;nsid=109867591;c=1731974958292), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:09:18,929 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073
2024-11-19T00:09:18,931 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/cluster_4aa9ef7b-a731-47c8-baac-55880e8d45de/zookeeper_0, clientPort=49738, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/cluster_4aa9ef7b-a731-47c8-baac-55880e8d45de/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/cluster_4aa9ef7b-a731-47c8-baac-55880e8d45de/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-19T00:09:18,932 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49738
2024-11-19T00:09:18,932 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:09:18,934 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:09:18,941 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741825_1001 (size=7)
2024-11-19T00:09:18,941 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741825_1001 (size=7)
2024-11-19T00:09:18,942 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9 with version=8
2024-11-19T00:09:18,942 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/hbase-staging
2024-11-19T00:09:18,944 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/9c732c6543ff:0 server-side Connection retries=45
2024-11-19T00:09:18,944 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:09:18,944 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T00:09:18,944 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T00:09:18,944 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:09:18,944 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T00:09:18,944 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-19T00:09:18,945 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T00:09:18,945 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40849
2024-11-19T00:09:18,946 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40849 connecting to ZooKeeper ensemble=127.0.0.1:49738
2024-11-19T00:09:18,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:408490x0, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T00:09:18,951 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40849-0x101085383fd0000 connected
2024-11-19T00:09:18,964 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:09:18,965 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:09:18,967 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:09:18,967 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9, hbase.cluster.distributed=false
2024-11-19T00:09:18,968 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T00:09:18,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40849
2024-11-19T00:09:18,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40849
2024-11-19T00:09:18,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40849
2024-11-19T00:09:18,970 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40849
2024-11-19T00:09:18,970 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40849
2024-11-19T00:09:18,985 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/9c732c6543ff:0 server-side Connection retries=45
2024-11-19T00:09:18,985 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:09:18,985 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T00:09:18,985 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T00:09:18,985 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:09:18,985 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T00:09:18,985 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-19T00:09:18,985 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T00:09:18,986 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43171
2024-11-19T00:09:18,987 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43171 connecting to ZooKeeper ensemble=127.0.0.1:49738
2024-11-19T00:09:18,988 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:09:18,989 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:09:18,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:431710x0, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T00:09:18,994 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43171-0x101085383fd0001 connected
2024-11-19T00:09:18,994 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:09:18,994 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-19T00:09:18,994 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-19T00:09:18,995 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-19T00:09:18,996 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T00:09:18,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43171
2024-11-19T00:09:18,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43171
2024-11-19T00:09:18,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43171
2024-11-19T00:09:18,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43171
2024-11-19T00:09:18,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43171
2024-11-19T00:09:19,009 DEBUG [M:0;9c732c6543ff:40849 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9c732c6543ff:40849
2024-11-19T00:09:19,009 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9c732c6543ff,40849,1731974958944
2024-11-19T00:09:19,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:09:19,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:09:19,011 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9c732c6543ff,40849,1731974958944
2024-11-19T00:09:19,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-19T00:09:19,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:09:19,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:09:19,013 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-19T00:09:19,013 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9c732c6543ff,40849,1731974958944 from backup master directory
2024-11-19T00:09:19,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9c732c6543ff,40849,1731974958944
2024-11-19T00:09:19,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:09:19,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:09:19,014 WARN  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T00:09:19,014 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9c732c6543ff,40849,1731974958944
2024-11-19T00:09:19,019 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/hbase.id] with ID: 1659dbaf-8632-4e9d-a3c3-e867990274dc
2024-11-19T00:09:19,019 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/.tmp/hbase.id
2024-11-19T00:09:19,025 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741826_1002 (size=42)
2024-11-19T00:09:19,025 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741826_1002 (size=42)
2024-11-19T00:09:19,026 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/.tmp/hbase.id]:[hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/hbase.id]
2024-11-19T00:09:19,035 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:09:19,036 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-19T00:09:19,037 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-19T00:09:19,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:09:19,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:09:19,046 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741827_1003 (size=196)
2024-11-19T00:09:19,046 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741827_1003 (size=196)
2024-11-19T00:09:19,047 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T00:09:19,048 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-19T00:09:19,048 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:09:19,054 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741828_1004 (size=1189)
2024-11-19T00:09:19,054 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741828_1004 (size=1189)
2024-11-19T00:09:19,055 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store
2024-11-19T00:09:19,061 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741829_1005 (size=34)
2024-11-19T00:09:19,061 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741829_1005 (size=34)
2024-11-19T00:09:19,062 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:09:19,062 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T00:09:19,062 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:09:19,062 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:09:19,062 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T00:09:19,062 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:09:19,062 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:09:19,062 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731974959062Disabling compacts and flushes for region at 1731974959062Disabling writes for close at 1731974959062Writing region close event to WAL at 1731974959062Closed at 1731974959062
2024-11-19T00:09:19,063 WARN  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/.initializing
2024-11-19T00:09:19,063 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/WALs/9c732c6543ff,40849,1731974958944
2024-11-19T00:09:19,065 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C40849%2C1731974958944, suffix=, logDir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/WALs/9c732c6543ff,40849,1731974958944, archiveDir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/oldWALs, maxLogs=10
2024-11-19T00:09:19,065 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C40849%2C1731974958944.1731974959065
2024-11-19T00:09:19,069 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/WALs/9c732c6543ff,40849,1731974958944/9c732c6543ff%2C40849%2C1731974958944.1731974959065
2024-11-19T00:09:19,073 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42451:42451),(127.0.0.1/127.0.0.1:36629:36629)]
2024-11-19T00:09:19,074 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:09:19,074 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:09:19,074 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:09:19,074 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:09:19,076 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:09:19,077 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-19T00:09:19,077 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:19,077 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:09:19,077 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:09:19,079 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-19T00:09:19,079 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:19,079 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:09:19,079 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:09:19,080 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-19T00:09:19,080 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:19,081 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:09:19,081 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:09:19,082 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-19T00:09:19,082 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:19,082 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:09:19,083 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:09:19,083 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:09:19,084 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:09:19,085 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:09:19,085 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:09:19,086 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-19T00:09:19,087 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:09:19,089 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:09:19,090 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=794782, jitterRate=0.010618001222610474}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-19T00:09:19,090 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731974959075Initializing all the Stores at 1731974959075Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974959075Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974959076 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974959076Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974959076Cleaning up temporary data from old regions at 1731974959085 (+9 ms)Region opened successfully at 1731974959090 (+5 ms)
2024-11-19T00:09:19,093 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-19T00:09:19,096 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@251d6732, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0
2024-11-19T00:09:19,097 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-19T00:09:19,097 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-19T00:09:19,097 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-19T00:09:19,097 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-19T00:09:19,098 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-19T00:09:19,098 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-19T00:09:19,098 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-19T00:09:19,100 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-19T00:09:19,101 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-19T00:09:19,103 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-19T00:09:19,103 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-19T00:09:19,104 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-19T00:09:19,105 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-19T00:09:19,106 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-19T00:09:19,107 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-19T00:09:19,109 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-19T00:09:19,110 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-19T00:09:19,111 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-19T00:09:19,113 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-19T00:09:19,115 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-19T00:09:19,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T00:09:19,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T00:09:19,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:09:19,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:09:19,117 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9c732c6543ff,40849,1731974958944, sessionid=0x101085383fd0000, setting cluster-up flag (Was=false)
2024-11-19T00:09:19,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:09:19,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:09:19,127 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-19T00:09:19,129 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,40849,1731974958944
2024-11-19T00:09:19,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:09:19,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:09:19,144 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-19T00:09:19,149 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,40849,1731974958944
2024-11-19T00:09:19,153 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-19T00:09:19,162 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-19T00:09:19,163 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-19T00:09:19,163 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-19T00:09:19,163 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9c732c6543ff,40849,1731974958944
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-19T00:09:19,165 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:09:19,165 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:09:19,165 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:09:19,165 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:09:19,165 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9c732c6543ff:0, corePoolSize=10, maxPoolSize=10
2024-11-19T00:09:19,165 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:09:19,165 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2
2024-11-19T00:09:19,165 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:09:19,185 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731974989185
2024-11-19T00:09:19,186 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-19T00:09:19,186 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-19T00:09:19,186 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-19T00:09:19,186 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-19T00:09:19,186 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-19T00:09:19,186 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-19T00:09:19,186 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,187 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-19T00:09:19,187 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-19T00:09:19,187 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-19T00:09:19,191 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:09:19,191 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-19T00:09:19,193 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:19,193 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-19T00:09:19,201 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-19T00:09:19,201 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-19T00:09:19,205 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974959201,5,FailOnTimeoutGroup]
2024-11-19T00:09:19,219 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974959205,5,FailOnTimeoutGroup]
2024-11-19T00:09:19,219 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,219 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-19T00:09:19,219 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,219 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,243 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer(746): ClusterId : 1659dbaf-8632-4e9d-a3c3-e867990274dc
2024-11-19T00:09:19,243 DEBUG [RS:0;9c732c6543ff:43171 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-19T00:09:19,247 DEBUG [RS:0;9c732c6543ff:43171 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-19T00:09:19,247 DEBUG [RS:0;9c732c6543ff:43171 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-19T00:09:19,251 DEBUG [RS:0;9c732c6543ff:43171 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-19T00:09:19,251 DEBUG [RS:0;9c732c6543ff:43171 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28f65885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0
2024-11-19T00:09:19,270 DEBUG [RS:0;9c732c6543ff:43171 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9c732c6543ff:43171
2024-11-19T00:09:19,270 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-19T00:09:19,270 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-19T00:09:19,270 DEBUG [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-19T00:09:19,271 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer(2659): reportForDuty to master=9c732c6543ff,40849,1731974958944 with port=43171, startcode=1731974958985
2024-11-19T00:09:19,271 DEBUG [RS:0;9c732c6543ff:43171 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-19T00:09:19,291 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741831_1007 (size=1321)
2024-11-19T00:09:19,291 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741831_1007 (size=1321)
2024-11-19T00:09:19,293 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-19T00:09:19,293 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9
2024-11-19T00:09:19,294 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44469, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-11-19T00:09:19,295 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40849 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9c732c6543ff,43171,1731974958985
2024-11-19T00:09:19,295 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40849 {}] master.ServerManager(517): Registering regionserver=9c732c6543ff,43171,1731974958985
2024-11-19T00:09:19,297 DEBUG [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9
2024-11-19T00:09:19,297 DEBUG [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40297
2024-11-19T00:09:19,297 DEBUG [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-19T00:09:19,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T00:09:19,299 DEBUG [RS:0;9c732c6543ff:43171 {}] zookeeper.ZKUtil(111): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9c732c6543ff,43171,1731974958985
2024-11-19T00:09:19,300 WARN  [RS:0;9c732c6543ff:43171 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T00:09:19,300 INFO  [RS:0;9c732c6543ff:43171 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:09:19,300 DEBUG [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/WALs/9c732c6543ff,43171,1731974958985
2024-11-19T00:09:19,306 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9c732c6543ff,43171,1731974958985]
2024-11-19T00:09:19,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741832_1008 (size=32)
2024-11-19T00:09:19,320 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741832_1008 (size=32)
2024-11-19T00:09:19,320 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:09:19,321 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T00:09:19,322 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-19T00:09:19,323 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T00:09:19,323 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:19,325 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-19T00:09:19,325 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:09:19,325 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T00:09:19,327 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T00:09:19,327 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:19,328 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:09:19,328 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T00:09:19,328 INFO  [RS:0;9c732c6543ff:43171 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T00:09:19,328 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,328 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-19T00:09:19,329 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T00:09:19,329 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:19,329 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:09:19,329 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T00:09:19,330 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-19T00:09:19,330 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,330 DEBUG [RS:0;9c732c6543ff:43171 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:09:19,330 DEBUG [RS:0;9c732c6543ff:43171 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:09:19,330 DEBUG [RS:0;9c732c6543ff:43171 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:09:19,330 DEBUG [RS:0;9c732c6543ff:43171 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:09:19,330 DEBUG [RS:0;9c732c6543ff:43171 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:09:19,330 DEBUG [RS:0;9c732c6543ff:43171 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2
2024-11-19T00:09:19,330 DEBUG [RS:0;9c732c6543ff:43171 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:09:19,330 DEBUG [RS:0;9c732c6543ff:43171 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:09:19,330 DEBUG [RS:0;9c732c6543ff:43171 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:09:19,330 DEBUG [RS:0;9c732c6543ff:43171 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:09:19,330 DEBUG [RS:0;9c732c6543ff:43171 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:09:19,330 DEBUG [RS:0;9c732c6543ff:43171 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:09:19,330 DEBUG [RS:0;9c732c6543ff:43171 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3
2024-11-19T00:09:19,330 DEBUG [RS:0;9c732c6543ff:43171 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3
2024-11-19T00:09:19,331 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T00:09:19,331 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:19,332 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:09:19,332 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T00:09:19,333 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740
2024-11-19T00:09:19,333 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740
2024-11-19T00:09:19,333 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,333 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,333 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,333 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,333 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,333 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,43171,1731974958985-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T00:09:19,334 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T00:09:19,334 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T00:09:19,335 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T00:09:19,337 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T00:09:19,342 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:09:19,342 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=862986, jitterRate=0.09734432399272919}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T00:09:19,343 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731974959320Initializing all the Stores at 1731974959321 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974959321Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974959321Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974959321Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974959321Cleaning up temporary data from old regions at 1731974959334 (+13 ms)Region opened successfully at 1731974959343 (+9 ms)
2024-11-19T00:09:19,343 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T00:09:19,343 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T00:09:19,343 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T00:09:19,343 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T00:09:19,343 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T00:09:19,348 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T00:09:19,348 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731974959343Disabling compacts and flushes for region at 1731974959343Disabling writes for close at 1731974959343Writing region close event to WAL at 1731974959348 (+5 ms)Closed at 1731974959348
2024-11-19T00:09:19,349 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:09:19,349 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-19T00:09:19,350 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-19T00:09:19,351 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T00:09:19,353 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-19T00:09:19,354 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-19T00:09:19,355 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,43171,1731974958985-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,355 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,355 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.Replication(171): 9c732c6543ff,43171,1731974958985 started
2024-11-19T00:09:19,374 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,374 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer(1482): Serving as 9c732c6543ff,43171,1731974958985, RpcServer on 9c732c6543ff/172.17.0.2:43171, sessionid=0x101085383fd0001
2024-11-19T00:09:19,375 DEBUG [RS:0;9c732c6543ff:43171 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-19T00:09:19,375 DEBUG [RS:0;9c732c6543ff:43171 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9c732c6543ff,43171,1731974958985
2024-11-19T00:09:19,375 DEBUG [RS:0;9c732c6543ff:43171 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,43171,1731974958985'
2024-11-19T00:09:19,375 DEBUG [RS:0;9c732c6543ff:43171 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-19T00:09:19,375 DEBUG [RS:0;9c732c6543ff:43171 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-19T00:09:19,376 DEBUG [RS:0;9c732c6543ff:43171 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-19T00:09:19,376 DEBUG [RS:0;9c732c6543ff:43171 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-19T00:09:19,376 DEBUG [RS:0;9c732c6543ff:43171 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9c732c6543ff,43171,1731974958985
2024-11-19T00:09:19,376 DEBUG [RS:0;9c732c6543ff:43171 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,43171,1731974958985'
2024-11-19T00:09:19,376 DEBUG [RS:0;9c732c6543ff:43171 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-19T00:09:19,376 DEBUG [RS:0;9c732c6543ff:43171 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-19T00:09:19,377 DEBUG [RS:0;9c732c6543ff:43171 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-19T00:09:19,377 INFO  [RS:0;9c732c6543ff:43171 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-19T00:09:19,377 INFO  [RS:0;9c732c6543ff:43171 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-19T00:09:19,479 INFO  [RS:0;9c732c6543ff:43171 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C43171%2C1731974958985, suffix=, logDir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/WALs/9c732c6543ff,43171,1731974958985, archiveDir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/oldWALs, maxLogs=32
2024-11-19T00:09:19,479 INFO  [RS:0;9c732c6543ff:43171 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C43171%2C1731974958985.1731974959479
2024-11-19T00:09:19,492 INFO  [RS:0;9c732c6543ff:43171 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/WALs/9c732c6543ff,43171,1731974958985/9c732c6543ff%2C43171%2C1731974958985.1731974959479
2024-11-19T00:09:19,503 WARN  [9c732c6543ff:40849 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-19T00:09:19,505 DEBUG [RS:0;9c732c6543ff:43171 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42451:42451),(127.0.0.1/127.0.0.1:36629:36629)]
2024-11-19T00:09:19,753 DEBUG [9c732c6543ff:40849 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-19T00:09:19,754 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9c732c6543ff,43171,1731974958985
2024-11-19T00:09:19,756 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,43171,1731974958985, state=OPENING
2024-11-19T00:09:19,757 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:19,758 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-19T00:09:19,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:09:19,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:09:19,760 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T00:09:19,760 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:09:19,760 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,43171,1731974958985}]
2024-11-19T00:09:19,761 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:09:19,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:19,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:19,914 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-19T00:09:19,916 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43431, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-19T00:09:19,920 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-19T00:09:19,920 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:09:19,922 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C43171%2C1731974958985.meta, suffix=.meta, logDir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/WALs/9c732c6543ff,43171,1731974958985, archiveDir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/oldWALs, maxLogs=32
2024-11-19T00:09:19,923 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C43171%2C1731974958985.meta.1731974959922.meta
2024-11-19T00:09:19,930 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/WALs/9c732c6543ff,43171,1731974958985/9c732c6543ff%2C43171%2C1731974958985.meta.1731974959922.meta
2024-11-19T00:09:19,936 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36629:36629),(127.0.0.1/127.0.0.1:42451:42451)]
2024-11-19T00:09:19,936 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:09:19,937 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-19T00:09:19,937 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-19T00:09:19,937 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-19T00:09:19,937 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-19T00:09:19,937 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:09:19,937 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-19T00:09:19,937 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-19T00:09:19,940 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T00:09:19,941 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T00:09:19,941 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:19,941 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:09:19,941 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T00:09:19,942 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T00:09:19,942 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:19,943 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:09:19,943 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T00:09:19,943 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T00:09:19,943 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:19,944 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:09:19,944 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T00:09:19,945 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T00:09:19,945 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:19,945 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:09:19,945 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T00:09:19,946 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740
2024-11-19T00:09:19,947 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740
2024-11-19T00:09:19,948 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T00:09:19,948 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T00:09:19,949 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T00:09:19,950 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T00:09:19,951 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=790394, jitterRate=0.005038157105445862}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T00:09:19,951 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-19T00:09:19,952 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731974959937Writing region info on filesystem at 1731974959937Initializing all the Stores at 1731974959938 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974959938Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974959940 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974959940Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974959940Cleaning up temporary data from old regions at 1731974959948 (+8 ms)Running coprocessor post-open hooks at 1731974959951 (+3 ms)Region opened successfully at 1731974959952 (+1 ms)
2024-11-19T00:09:19,953 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731974959913
2024-11-19T00:09:19,956 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-19T00:09:19,956 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-19T00:09:19,957 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,43171,1731974958985
2024-11-19T00:09:19,958 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,43171,1731974958985, state=OPEN
2024-11-19T00:09:19,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T00:09:19,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T00:09:19,962 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:09:19,962 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:09:19,962 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9c732c6543ff,43171,1731974958985
2024-11-19T00:09:19,967 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-19T00:09:19,968 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,43171,1731974958985 in 202 msec
2024-11-19T00:09:19,970 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-19T00:09:19,970 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 619 msec
2024-11-19T00:09:19,971 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:09:19,971 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-19T00:09:19,973 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T00:09:19,973 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,43171,1731974958985, seqNum=-1]
2024-11-19T00:09:19,973 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T00:09:19,974 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41149, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T00:09:19,980 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 821 msec
2024-11-19T00:09:19,980 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731974959980, completionTime=-1
2024-11-19T00:09:19,980 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-19T00:09:19,980 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-19T00:09:19,982 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-19T00:09:19,982 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731975019982
2024-11-19T00:09:19,982 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731975079982
2024-11-19T00:09:19,982 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-19T00:09:19,982 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,40849,1731974958944-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,983 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,40849,1731974958944-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,983 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,40849,1731974958944-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,983 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9c732c6543ff:40849, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,983 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,985 DEBUG [master/9c732c6543ff:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-19T00:09:19,985 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:19,989 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.974sec
2024-11-19T00:09:19,989 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-19T00:09:19,989 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-19T00:09:19,989 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-19T00:09:19,989 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-19T00:09:19,989 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-19T00:09:19,989 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,40849,1731974958944-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T00:09:19,989 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,40849,1731974958944-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-19T00:09:19,992 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-19T00:09:19,992 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-19T00:09:19,992 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,40849,1731974958944-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:09:20,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e48481, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:09:20,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9c732c6543ff,40849,-1 for getting cluster id
2024-11-19T00:09:20,044 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-19T00:09:20,045 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659dbaf-8632-4e9d-a3c3-e867990274dc'
2024-11-19T00:09:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-19T00:09:20,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659dbaf-8632-4e9d-a3c3-e867990274dc"

2024-11-19T00:09:20,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cc7c111, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:09:20,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9c732c6543ff,40849,-1]
2024-11-19T00:09:20,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-19T00:09:20,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:09:20,048 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-19T00:09:20,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f6e773d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:09:20,049 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T00:09:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,43171,1731974958985, seqNum=-1]
2024-11-19T00:09:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T00:09:20,052 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T00:09:20,053 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9c732c6543ff,40849,1731974958944
2024-11-19T00:09:20,054 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:09:20,057 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-19T00:09:20,057 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-19T00:09:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 9c732c6543ff,40849,1731974958944
2024-11-19T00:09:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3f899f81
2024-11-19T00:09:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-19T00:09:20,059 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-19T00:09:20,060 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40849 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-19T00:09:20,060 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40849 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-19T00:09:20,060 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40849 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T00:09:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40849 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-11-19T00:09:20,063 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-19T00:09:20,063 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:20,063 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40849 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-11-19T00:09:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40849 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T00:09:20,065 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-19T00:09:20,073 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741835_1011 (size=381)
2024-11-19T00:09:20,073 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741835_1011 (size=381)
2024-11-19T00:09:20,075 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5f64dec1279ff8bc85e4392992959c31, NAME => 'TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9
2024-11-19T00:09:20,082 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741836_1012 (size=64)
2024-11-19T00:09:20,082 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741836_1012 (size=64)
2024-11-19T00:09:20,083 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:09:20,083 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 5f64dec1279ff8bc85e4392992959c31, disabling compactions & flushes
2024-11-19T00:09:20,083 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.
2024-11-19T00:09:20,083 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.
2024-11-19T00:09:20,083 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31. after waiting 0 ms
2024-11-19T00:09:20,083 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.
2024-11-19T00:09:20,083 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.
2024-11-19T00:09:20,083 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5f64dec1279ff8bc85e4392992959c31:
Waiting for close lock at 1731974960083Disabling compacts and flushes for region at 1731974960083Disabling writes for close at 1731974960083Writing region close event to WAL at 1731974960083Closed at 1731974960083
2024-11-19T00:09:20,085 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-19T00:09:20,085 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731974960085"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731974960085"}]},"ts":"1731974960085"}
2024-11-19T00:09:20,088 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-19T00:09:20,090 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-19T00:09:20,090 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731974960090"}]},"ts":"1731974960090"}
2024-11-19T00:09:20,093 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-11-19T00:09:20,093 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5f64dec1279ff8bc85e4392992959c31, ASSIGN}]
2024-11-19T00:09:20,094 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5f64dec1279ff8bc85e4392992959c31, ASSIGN
2024-11-19T00:09:20,095 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5f64dec1279ff8bc85e4392992959c31, ASSIGN; state=OFFLINE, location=9c732c6543ff,43171,1731974958985; forceNewPlan=false, retain=false
2024-11-19T00:09:20,246 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5f64dec1279ff8bc85e4392992959c31, regionState=OPENING, regionLocation=9c732c6543ff,43171,1731974958985
2024-11-19T00:09:20,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5f64dec1279ff8bc85e4392992959c31, ASSIGN because future has completed
2024-11-19T00:09:20,250 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f64dec1279ff8bc85e4392992959c31, server=9c732c6543ff,43171,1731974958985}]
2024-11-19T00:09:20,407 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.
2024-11-19T00:09:20,407 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f64dec1279ff8bc85e4392992959c31, NAME => 'TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:09:20,408 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:20,408 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:09:20,408 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:20,408 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:20,409 INFO  [StoreOpener-5f64dec1279ff8bc85e4392992959c31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 5f64dec1279ff8bc85e4392992959c31 
2024-11-19T00:09:20,411 INFO  [StoreOpener-5f64dec1279ff8bc85e4392992959c31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f64dec1279ff8bc85e4392992959c31 columnFamilyName info
2024-11-19T00:09:20,411 DEBUG [StoreOpener-5f64dec1279ff8bc85e4392992959c31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:20,415 INFO  [StoreOpener-5f64dec1279ff8bc85e4392992959c31-1 {}] regionserver.HStore(327): Store=5f64dec1279ff8bc85e4392992959c31/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:09:20,415 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:20,416 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:20,416 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:20,417 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:20,417 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:20,419 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:20,422 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:09:20,423 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5f64dec1279ff8bc85e4392992959c31; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=717349, jitterRate=-0.0878448337316513}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-19T00:09:20,423 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:20,424 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5f64dec1279ff8bc85e4392992959c31:
Running coprocessor pre-open hook at 1731974960408Writing region info on filesystem at 1731974960408Initializing all the Stores at 1731974960409 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974960409Cleaning up temporary data from old regions at 1731974960417 (+8 ms)Running coprocessor post-open hooks at 1731974960423 (+6 ms)Region opened successfully at 1731974960423
2024-11-19T00:09:20,425 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31., pid=6, masterSystemTime=1731974960403
2024-11-19T00:09:20,428 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.
2024-11-19T00:09:20,428 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.
2024-11-19T00:09:20,428 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5f64dec1279ff8bc85e4392992959c31, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,43171,1731974958985
2024-11-19T00:09:20,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f64dec1279ff8bc85e4392992959c31, server=9c732c6543ff,43171,1731974958985 because future has completed
2024-11-19T00:09:20,434 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-19T00:09:20,435 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f64dec1279ff8bc85e4392992959c31, server=9c732c6543ff,43171,1731974958985 in 182 msec
2024-11-19T00:09:20,438 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-19T00:09:20,438 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5f64dec1279ff8bc85e4392992959c31, ASSIGN in 342 msec
2024-11-19T00:09:20,439 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-19T00:09:20,439 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731974960439"}]},"ts":"1731974960439"}
2024-11-19T00:09:20,441 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-11-19T00:09:20,442 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-19T00:09:20,445 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 383 msec
2024-11-19T00:09:20,758 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:20,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:20,821 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:21,758 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:21,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:21,821 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:22,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:22,791 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:22,791 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:22,792 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:22,792 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:22,792 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:22,792 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:22,793 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:22,793 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:22,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:22,817 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:22,818 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:22,818 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:22,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:22,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:22,820 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:22,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:22,824 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:22,824 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:22,824 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:22,827 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,333 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T00:09:23,334 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,335 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,335 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,335 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,336 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,336 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,337 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,337 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,367 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,368 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,368 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,368 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,368 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,369 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,374 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,374 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,375 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,378 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:23,760 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:23,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:23,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:24,760 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:24,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:24,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:25,322 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-19T00:09:25,323 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-11-19T00:09:25,761 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:25,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:25,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:26,253 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-19T00:09:26,253 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-19T00:09:26,254 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T00:09:26,762 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:26,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:26,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:27,762 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:27,817 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:27,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:28,763 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:28,817 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:28,825 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:29,764 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:29,818 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:29,825 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40849 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T00:09:30,084 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-11-19T00:09:30,084 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-11-19T00:09:30,087 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-11-19T00:09:30,087 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.
2024-11-19T00:09:30,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31., hostname=9c732c6543ff,43171,1731974958985, seqNum=2]
2024-11-19T00:09:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:30,106 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f64dec1279ff8bc85e4392992959c31 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T00:09:30,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/1f09a515de7a463fb9f56653745c23b4 is 1080, key is row0001/info:/1731974970092/Put/seqid=0
2024-11-19T00:09:30,138 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741837_1013 (size=12509)
2024-11-19T00:09:30,138 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741837_1013 (size=12509)
2024-11-19T00:09:30,139 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/1f09a515de7a463fb9f56653745c23b4
2024-11-19T00:09:30,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/1f09a515de7a463fb9f56653745c23b4 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/1f09a515de7a463fb9f56653745c23b4
2024-11-19T00:09:30,155 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/1f09a515de7a463fb9f56653745c23b4, entries=7, sequenceid=11, filesize=12.2 K
2024-11-19T00:09:30,156 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 5f64dec1279ff8bc85e4392992959c31 in 51ms, sequenceid=11, compaction requested=false
2024-11-19T00:09:30,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f64dec1279ff8bc85e4392992959c31:

2024-11-19T00:09:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:30,157 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f64dec1279ff8bc85e4392992959c31 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-11-19T00:09:30,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/d15a3423f92b4c2086ea5daefd4545d8 is 1080, key is row0008/info:/1731974970106/Put/seqid=0
2024-11-19T00:09:30,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741838_1014 (size=29761)
2024-11-19T00:09:30,172 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741838_1014 (size=29761)
2024-11-19T00:09:30,173 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/d15a3423f92b4c2086ea5daefd4545d8
2024-11-19T00:09:30,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/d15a3423f92b4c2086ea5daefd4545d8 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/d15a3423f92b4c2086ea5daefd4545d8
2024-11-19T00:09:30,184 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/d15a3423f92b4c2086ea5daefd4545d8, entries=23, sequenceid=37, filesize=29.1 K
2024-11-19T00:09:30,185 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for 5f64dec1279ff8bc85e4392992959c31 in 28ms, sequenceid=37, compaction requested=false
2024-11-19T00:09:30,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f64dec1279ff8bc85e4392992959c31:

2024-11-19T00:09:30,186 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=41.3 K, sizeToCheck=16.0 K
2024-11-19T00:09:30,186 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:09:30,186 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/d15a3423f92b4c2086ea5daefd4545d8 because midkey is the same as first or last row
2024-11-19T00:09:30,764 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:30,818 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:30,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:31,756 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T00:09:31,757 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,758 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,758 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,758 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,758 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,759 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,759 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,760 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,765 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:31,790 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,791 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,791 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,791 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,791 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,792 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,796 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,797 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,797 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,800 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:31,819 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:31,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:32,174 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f64dec1279ff8bc85e4392992959c31 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T00:09:32,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/0c07b7b0dc7a4791b7c2fe9dcc1692bd is 1080, key is row0031/info:/1731974970159/Put/seqid=0
2024-11-19T00:09:32,189 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741839_1015 (size=12509)
2024-11-19T00:09:32,190 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741839_1015 (size=12509)
2024-11-19T00:09:32,191 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/0c07b7b0dc7a4791b7c2fe9dcc1692bd
2024-11-19T00:09:32,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/0c07b7b0dc7a4791b7c2fe9dcc1692bd as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/0c07b7b0dc7a4791b7c2fe9dcc1692bd
2024-11-19T00:09:32,205 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/0c07b7b0dc7a4791b7c2fe9dcc1692bd, entries=7, sequenceid=47, filesize=12.2 K
2024-11-19T00:09:32,218 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for 5f64dec1279ff8bc85e4392992959c31 in 43ms, sequenceid=47, compaction requested=true
2024-11-19T00:09:32,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f64dec1279ff8bc85e4392992959c31:

2024-11-19T00:09:32,218 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=53.5 K, sizeToCheck=16.0 K
2024-11-19T00:09:32,218 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:09:32,218 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/d15a3423f92b4c2086ea5daefd4545d8 because midkey is the same as first or last row
2024-11-19T00:09:32,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f64dec1279ff8bc85e4392992959c31:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T00:09:32,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:09:32,219 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T00:09:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:32,221 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f64dec1279ff8bc85e4392992959c31 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-19T00:09:32,221 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T00:09:32,221 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1541): 5f64dec1279ff8bc85e4392992959c31/info is initiating minor compaction (all files)
2024-11-19T00:09:32,222 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f64dec1279ff8bc85e4392992959c31/info in TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.
2024-11-19T00:09:32,222 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/1f09a515de7a463fb9f56653745c23b4, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/d15a3423f92b4c2086ea5daefd4545d8, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/0c07b7b0dc7a4791b7c2fe9dcc1692bd] into tmpdir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp, totalSize=53.5 K
2024-11-19T00:09:32,222 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f09a515de7a463fb9f56653745c23b4, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731974970092
2024-11-19T00:09:32,224 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting d15a3423f92b4c2086ea5daefd4545d8, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731974970106
2024-11-19T00:09:32,225 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c07b7b0dc7a4791b7c2fe9dcc1692bd, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731974970159
2024-11-19T00:09:32,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/9212a01120ab483499addeb2ca561f80 is 1080, key is row0038/info:/1731974972175/Put/seqid=0
2024-11-19T00:09:32,260 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f64dec1279ff8bc85e4392992959c31#info#compaction#60 average throughput is 18.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T00:09:32,260 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/93861b513ab940f68223618e991286c2 is 1080, key is row0001/info:/1731974970092/Put/seqid=0
2024-11-19T00:09:32,267 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741840_1016 (size=17894)
2024-11-19T00:09:32,269 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741840_1016 (size=17894)
2024-11-19T00:09:32,270 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/9212a01120ab483499addeb2ca561f80
2024-11-19T00:09:32,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/9212a01120ab483499addeb2ca561f80 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/9212a01120ab483499addeb2ca561f80
2024-11-19T00:09:32,286 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/9212a01120ab483499addeb2ca561f80, entries=12, sequenceid=62, filesize=17.5 K
2024-11-19T00:09:32,289 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=15.76 KB/16140 for 5f64dec1279ff8bc85e4392992959c31 in 68ms, sequenceid=62, compaction requested=false
2024-11-19T00:09:32,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f64dec1279ff8bc85e4392992959c31:

2024-11-19T00:09:32,289 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=71.0 K, sizeToCheck=16.0 K
2024-11-19T00:09:32,289 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:09:32,289 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/d15a3423f92b4c2086ea5daefd4545d8 because midkey is the same as first or last row
2024-11-19T00:09:32,298 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741841_1017 (size=44978)
2024-11-19T00:09:32,300 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741841_1017 (size=44978)
2024-11-19T00:09:32,308 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/93861b513ab940f68223618e991286c2 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/93861b513ab940f68223618e991286c2
2024-11-19T00:09:32,317 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f64dec1279ff8bc85e4392992959c31/info of 5f64dec1279ff8bc85e4392992959c31 into 93861b513ab940f68223618e991286c2(size=43.9 K), total size for store is 61.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T00:09:32,317 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f64dec1279ff8bc85e4392992959c31:

2024-11-19T00:09:32,318 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31., storeName=5f64dec1279ff8bc85e4392992959c31/info, priority=13, startTime=1731974972218; duration=0sec
2024-11-19T00:09:32,318 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=61.4 K, sizeToCheck=16.0 K
2024-11-19T00:09:32,318 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:09:32,318 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/93861b513ab940f68223618e991286c2 because midkey is the same as first or last row
2024-11-19T00:09:32,318 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=61.4 K, sizeToCheck=16.0 K
2024-11-19T00:09:32,318 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:09:32,318 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/93861b513ab940f68223618e991286c2 because midkey is the same as first or last row
2024-11-19T00:09:32,318 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=61.4 K, sizeToCheck=16.0 K
2024-11-19T00:09:32,318 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:09:32,318 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/93861b513ab940f68223618e991286c2 because midkey is the same as first or last row
2024-11-19T00:09:32,318 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:09:32,318 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f64dec1279ff8bc85e4392992959c31:info
2024-11-19T00:09:32,765 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:32,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:32,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:33,766 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:33,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:33,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:34,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:34,259 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f64dec1279ff8bc85e4392992959c31 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-11-19T00:09:34,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/13e960acdf254c91a1c3f414cd735bc7 is 1080, key is row0050/info:/1731974972223/Put/seqid=0
2024-11-19T00:09:34,272 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741842_1018 (size=22222)
2024-11-19T00:09:34,273 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741842_1018 (size=22222)
2024-11-19T00:09:34,273 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/13e960acdf254c91a1c3f414cd735bc7
2024-11-19T00:09:34,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/13e960acdf254c91a1c3f414cd735bc7 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/13e960acdf254c91a1c3f414cd735bc7
2024-11-19T00:09:34,286 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/13e960acdf254c91a1c3f414cd735bc7, entries=16, sequenceid=82, filesize=21.7 K
2024-11-19T00:09:34,287 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=9.46 KB/9684 for 5f64dec1279ff8bc85e4392992959c31 in 28ms, sequenceid=82, compaction requested=true
2024-11-19T00:09:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f64dec1279ff8bc85e4392992959c31:

2024-11-19T00:09:34,288 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=83.1 K, sizeToCheck=16.0 K
2024-11-19T00:09:34,288 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:09:34,288 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/93861b513ab940f68223618e991286c2 because midkey is the same as first or last row
2024-11-19T00:09:34,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f64dec1279ff8bc85e4392992959c31:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T00:09:34,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:09:34,288 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T00:09:34,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:34,289 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f64dec1279ff8bc85e4392992959c31 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-19T00:09:34,289 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 85094 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T00:09:34,289 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1541): 5f64dec1279ff8bc85e4392992959c31/info is initiating minor compaction (all files)
2024-11-19T00:09:34,289 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f64dec1279ff8bc85e4392992959c31/info in TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.
2024-11-19T00:09:34,289 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/93861b513ab940f68223618e991286c2, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/9212a01120ab483499addeb2ca561f80, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/13e960acdf254c91a1c3f414cd735bc7] into tmpdir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp, totalSize=83.1 K
2024-11-19T00:09:34,290 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93861b513ab940f68223618e991286c2, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731974970092
2024-11-19T00:09:34,291 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9212a01120ab483499addeb2ca561f80, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731974972175
2024-11-19T00:09:34,291 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13e960acdf254c91a1c3f414cd735bc7, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731974972223
2024-11-19T00:09:34,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/9cee2088f42049bdbe5d46dde8af5ef8 is 1080, key is row0066/info:/1731974974260/Put/seqid=0
2024-11-19T00:09:34,311 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741843_1019 (size=15740)
2024-11-19T00:09:34,312 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741843_1019 (size=15740)
2024-11-19T00:09:34,312 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/9cee2088f42049bdbe5d46dde8af5ef8
2024-11-19T00:09:34,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/9cee2088f42049bdbe5d46dde8af5ef8 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/9cee2088f42049bdbe5d46dde8af5ef8
2024-11-19T00:09:34,328 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f64dec1279ff8bc85e4392992959c31#info#compaction#63 average throughput is 33.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T00:09:34,328 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/768601cfc5ea4e6c8fd7f2e26d010790 is 1080, key is row0001/info:/1731974970092/Put/seqid=0
2024-11-19T00:09:34,330 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/9cee2088f42049bdbe5d46dde8af5ef8, entries=10, sequenceid=95, filesize=15.4 K
2024-11-19T00:09:34,331 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=17.86 KB/18292 for 5f64dec1279ff8bc85e4392992959c31 in 43ms, sequenceid=95, compaction requested=false
2024-11-19T00:09:34,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f64dec1279ff8bc85e4392992959c31:

2024-11-19T00:09:34,331 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=98.5 K, sizeToCheck=16.0 K
2024-11-19T00:09:34,331 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:09:34,331 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/93861b513ab940f68223618e991286c2 because midkey is the same as first or last row
2024-11-19T00:09:34,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:34,332 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f64dec1279ff8bc85e4392992959c31 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB
2024-11-19T00:09:34,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/5341192a871b4fbfae47647135d6345b is 1080, key is row0076/info:/1731974974291/Put/seqid=0
2024-11-19T00:09:34,343 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741844_1020 (size=75378)
2024-11-19T00:09:34,343 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741844_1020 (size=75378)
2024-11-19T00:09:34,351 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/768601cfc5ea4e6c8fd7f2e26d010790 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/768601cfc5ea4e6c8fd7f2e26d010790
2024-11-19T00:09:34,360 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f64dec1279ff8bc85e4392992959c31/info of 5f64dec1279ff8bc85e4392992959c31 into 768601cfc5ea4e6c8fd7f2e26d010790(size=73.6 K), total size for store is 89.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T00:09:34,360 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f64dec1279ff8bc85e4392992959c31:

2024-11-19T00:09:34,360 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31., storeName=5f64dec1279ff8bc85e4392992959c31/info, priority=13, startTime=1731974974288; duration=0sec
2024-11-19T00:09:34,360 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=89.0 K, sizeToCheck=16.0 K
2024-11-19T00:09:34,360 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:09:34,360 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=89.0 K, sizeToCheck=16.0 K
2024-11-19T00:09:34,360 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:09:34,361 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=89.0 K, sizeToCheck=16.0 K
2024-11-19T00:09:34,361 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:09:34,363 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741845_1021 (size=24376)
2024-11-19T00:09:34,363 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741845_1021 (size=24376)
2024-11-19T00:09:34,364 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/5341192a871b4fbfae47647135d6345b
2024-11-19T00:09:34,369 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:09:34,369 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:09:34,369 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f64dec1279ff8bc85e4392992959c31:info
2024-11-19T00:09:34,371 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40849 {}] assignment.AssignmentManager(1363): Split request from 9c732c6543ff,43171,1731974958985, parent={ENCODED => 5f64dec1279ff8bc85e4392992959c31, NAME => 'TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-11-19T00:09:34,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/5341192a871b4fbfae47647135d6345b as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/5341192a871b4fbfae47647135d6345b
2024-11-19T00:09:34,377 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40849 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=9c732c6543ff,43171,1731974958985
2024-11-19T00:09:34,380 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/5341192a871b4fbfae47647135d6345b, entries=18, sequenceid=116, filesize=23.8 K
2024-11-19T00:09:34,382 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=3.15 KB/3228 for 5f64dec1279ff8bc85e4392992959c31 in 50ms, sequenceid=116, compaction requested=true
2024-11-19T00:09:34,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f64dec1279ff8bc85e4392992959c31:

2024-11-19T00:09:34,382 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=112.8 K, sizeToCheck=16.0 K
2024-11-19T00:09:34,382 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:09:34,382 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=112.8 K, sizeToCheck=16.0 K
2024-11-19T00:09:34,382 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:09:34,382 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=112.8 K, sizeToCheck=16.0 K
2024-11-19T00:09:34,382 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T00:09:34,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=1
2024-11-19T00:09:34,382 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40849 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=5f64dec1279ff8bc85e4392992959c31, daughterA=cab914c3a7324c032510f1518a7f8646, daughterB=bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:34,384 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=5f64dec1279ff8bc85e4392992959c31, daughterA=cab914c3a7324c032510f1518a7f8646, daughterB=bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:34,384 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=5f64dec1279ff8bc85e4392992959c31, daughterA=cab914c3a7324c032510f1518a7f8646, daughterB=bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:34,384 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=5f64dec1279ff8bc85e4392992959c31, daughterA=cab914c3a7324c032510f1518a7f8646, daughterB=bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:34,389 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40849 {}] assignment.AssignmentManager(1363): Split request from 9c732c6543ff,43171,1731974958985, parent={ENCODED => 5f64dec1279ff8bc85e4392992959c31, NAME => 'TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-11-19T00:09:34,389 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40849 {}] assignment.AssignmentManager(1378): Ignoring split request from 9c732c6543ff,43171,1731974958985, parent={ENCODED => 5f64dec1279ff8bc85e4392992959c31, NAME => 'TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.', STARTKEY => '', ENDKEY => ''} because parent is unknown or not open
2024-11-19T00:09:34,393 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5f64dec1279ff8bc85e4392992959c31, UNASSIGN}]
2024-11-19T00:09:34,394 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5f64dec1279ff8bc85e4392992959c31, UNASSIGN
2024-11-19T00:09:34,395 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5f64dec1279ff8bc85e4392992959c31, regionState=CLOSING, regionLocation=9c732c6543ff,43171,1731974958985
2024-11-19T00:09:34,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5f64dec1279ff8bc85e4392992959c31, UNASSIGN because future has completed
2024-11-19T00:09:34,398 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-11-19T00:09:34,398 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5f64dec1279ff8bc85e4392992959c31, server=9c732c6543ff,43171,1731974958985}]
2024-11-19T00:09:34,556 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:34,556 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-11-19T00:09:34,556 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing 5f64dec1279ff8bc85e4392992959c31, disabling compactions & flushes
2024-11-19T00:09:34,556 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.
2024-11-19T00:09:34,556 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.
2024-11-19T00:09:34,556 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31. after waiting 0 ms
2024-11-19T00:09:34,557 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.
2024-11-19T00:09:34,557 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing 5f64dec1279ff8bc85e4392992959c31 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-11-19T00:09:34,562 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/0df66122f0bf4e3a9bcade95ecfb78d1 is 1080, key is row0094/info:/1731974974334/Put/seqid=0
2024-11-19T00:09:34,567 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741846_1022 (size=8193)
2024-11-19T00:09:34,569 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741846_1022 (size=8193)
2024-11-19T00:09:34,570 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/0df66122f0bf4e3a9bcade95ecfb78d1
2024-11-19T00:09:34,576 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/.tmp/info/0df66122f0bf4e3a9bcade95ecfb78d1 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/0df66122f0bf4e3a9bcade95ecfb78d1
2024-11-19T00:09:34,581 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/0df66122f0bf4e3a9bcade95ecfb78d1, entries=3, sequenceid=123, filesize=8.0 K
2024-11-19T00:09:34,582 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 5f64dec1279ff8bc85e4392992959c31 in 25ms, sequenceid=123, compaction requested=true
2024-11-19T00:09:34,584 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/1f09a515de7a463fb9f56653745c23b4, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/d15a3423f92b4c2086ea5daefd4545d8, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/93861b513ab940f68223618e991286c2, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/0c07b7b0dc7a4791b7c2fe9dcc1692bd, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/9212a01120ab483499addeb2ca561f80, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/13e960acdf254c91a1c3f414cd735bc7] to archive
2024-11-19T00:09:34,584 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-19T00:09:34,587 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/1f09a515de7a463fb9f56653745c23b4 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/1f09a515de7a463fb9f56653745c23b4
2024-11-19T00:09:34,589 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/d15a3423f92b4c2086ea5daefd4545d8 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/d15a3423f92b4c2086ea5daefd4545d8
2024-11-19T00:09:34,590 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/93861b513ab940f68223618e991286c2 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/93861b513ab940f68223618e991286c2
2024-11-19T00:09:34,592 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/0c07b7b0dc7a4791b7c2fe9dcc1692bd to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/0c07b7b0dc7a4791b7c2fe9dcc1692bd
2024-11-19T00:09:34,593 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/9212a01120ab483499addeb2ca561f80 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/9212a01120ab483499addeb2ca561f80
2024-11-19T00:09:34,594 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/13e960acdf254c91a1c3f414cd735bc7 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/13e960acdf254c91a1c3f414cd735bc7
2024-11-19T00:09:34,603 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=1
2024-11-19T00:09:34,604 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.
2024-11-19T00:09:34,604 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for 5f64dec1279ff8bc85e4392992959c31:
Waiting for close lock at 1731974974556Running coprocessor pre-close hooks at 1731974974556Disabling compacts and flushes for region at 1731974974556Disabling writes for close at 1731974974556Obtaining lock to block concurrent updates at 1731974974557 (+1 ms)Preparing flush snapshotting stores in 5f64dec1279ff8bc85e4392992959c31 at 1731974974557Finished memstore snapshotting TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1731974974557Flushing stores of TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31. at 1731974974558 (+1 ms)Flushing 5f64dec1279ff8bc85e4392992959c31/info: creating writer at 1731974974558Flushing 5f64dec1279ff8bc85e4392992959c31/info: appending metadata at 1731974974561 (+3 ms)Flushing 5f64dec1279ff8bc85e4392992959c31/info: closing flushed file at 1731974974561Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48f7087e: reopening flushed file at 1731974974575 (+14 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 5f64dec1279ff8bc85e4392992959c31 in 25ms, sequenceid=123, compaction requested=true at 1731974974582 (+7 ms)Writing region close event to WAL at 1731974974597 (+15 ms)Running coprocessor post-close hooks at 1731974974604 (+7 ms)Closed at 1731974974604
2024-11-19T00:09:34,607 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:34,607 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5f64dec1279ff8bc85e4392992959c31, regionState=CLOSED
2024-11-19T00:09:34,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5f64dec1279ff8bc85e4392992959c31, server=9c732c6543ff,43171,1731974958985 because future has completed
2024-11-19T00:09:34,614 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-19T00:09:34,614 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure 5f64dec1279ff8bc85e4392992959c31, server=9c732c6543ff,43171,1731974958985 in 214 msec
2024-11-19T00:09:34,617 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-19T00:09:34,617 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5f64dec1279ff8bc85e4392992959c31, UNASSIGN in 221 msec
2024-11-19T00:09:34,639 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:34,646 INFO  [PEWorker-3 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 4 storefiles, region=5f64dec1279ff8bc85e4392992959c31, threads=4
2024-11-19T00:09:34,649 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/0df66122f0bf4e3a9bcade95ecfb78d1 for region: 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:34,658 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/5341192a871b4fbfae47647135d6345b for region: 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:34,662 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/0df66122f0bf4e3a9bcade95ecfb78d1, top=true
2024-11-19T00:09:34,665 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/768601cfc5ea4e6c8fd7f2e26d010790 for region: 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:34,676 DEBUG [StoreFileSplitter-pool-3 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/9cee2088f42049bdbe5d46dde8af5ef8 for region: 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:34,677 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/5341192a871b4fbfae47647135d6345b, top=true
2024-11-19T00:09:34,694 DEBUG [StoreFileSplitter-pool-3 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/9cee2088f42049bdbe5d46dde8af5ef8, top=true
2024-11-19T00:09:34,698 INFO  [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-0df66122f0bf4e3a9bcade95ecfb78d1 for child: bc02cd12a280aa6311e039f4a9aaf2b1, parent: 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:34,698 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/0df66122f0bf4e3a9bcade95ecfb78d1 for region: 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:34,705 INFO  [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-5341192a871b4fbfae47647135d6345b for child: bc02cd12a280aa6311e039f4a9aaf2b1, parent: 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:34,705 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/5341192a871b4fbfae47647135d6345b for region: 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:34,727 INFO  [StoreFileSplitter-pool-3 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-9cee2088f42049bdbe5d46dde8af5ef8 for child: bc02cd12a280aa6311e039f4a9aaf2b1, parent: 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:34,727 DEBUG [StoreFileSplitter-pool-3 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/9cee2088f42049bdbe5d46dde8af5ef8 for region: 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:34,734 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741847_1023 (size=27)
2024-11-19T00:09:34,735 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741847_1023 (size=27)
2024-11-19T00:09:34,751 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741848_1024 (size=27)
2024-11-19T00:09:34,752 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741848_1024 (size=27)
2024-11-19T00:09:34,752 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/768601cfc5ea4e6c8fd7f2e26d010790 for region: 5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:09:34,754 DEBUG [PEWorker-3 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 5f64dec1279ff8bc85e4392992959c31 Daughter A: [hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/cab914c3a7324c032510f1518a7f8646/info/768601cfc5ea4e6c8fd7f2e26d010790.5f64dec1279ff8bc85e4392992959c31] storefiles, Daughter B: [hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-0df66122f0bf4e3a9bcade95ecfb78d1, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-5341192a871b4fbfae47647135d6345b, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/768601cfc5ea4e6c8fd7f2e26d010790.5f64dec1279ff8bc85e4392992959c31, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-9cee2088f42049bdbe5d46dde8af5ef8] storefiles.
2024-11-19T00:09:34,767 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:34,770 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741849_1025 (size=71)
2024-11-19T00:09:34,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741849_1025 (size=71)
2024-11-19T00:09:34,773 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:34,786 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741850_1026 (size=71)
2024-11-19T00:09:34,786 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741850_1026 (size=71)
2024-11-19T00:09:34,791 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:34,804 DEBUG [PEWorker-3 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/cab914c3a7324c032510f1518a7f8646/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=-1
2024-11-19T00:09:34,807 DEBUG [PEWorker-3 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=-1
2024-11-19T00:09:34,811 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731974974810"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1731974974810"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1731974974810"}]},"ts":"1731974974810"}
2024-11-19T00:09:34,811 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731974974810"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731974974810"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731974974810"}]},"ts":"1731974974810"}
2024-11-19T00:09:34,811 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731974974810"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731974974810"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731974974810"}]},"ts":"1731974974810"}
2024-11-19T00:09:34,821 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:34,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:34,833 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=cab914c3a7324c032510f1518a7f8646, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=bc02cd12a280aa6311e039f4a9aaf2b1, ASSIGN}]
2024-11-19T00:09:34,835 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=cab914c3a7324c032510f1518a7f8646, ASSIGN
2024-11-19T00:09:34,835 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=bc02cd12a280aa6311e039f4a9aaf2b1, ASSIGN
2024-11-19T00:09:34,836 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=cab914c3a7324c032510f1518a7f8646, ASSIGN; state=SPLITTING_NEW, location=9c732c6543ff,43171,1731974958985; forceNewPlan=false, retain=false
2024-11-19T00:09:34,837 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=bc02cd12a280aa6311e039f4a9aaf2b1, ASSIGN; state=SPLITTING_NEW, location=9c732c6543ff,43171,1731974958985; forceNewPlan=false, retain=false
2024-11-19T00:09:34,987 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=cab914c3a7324c032510f1518a7f8646, regionState=OPENING, regionLocation=9c732c6543ff,43171,1731974958985
2024-11-19T00:09:34,987 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=bc02cd12a280aa6311e039f4a9aaf2b1, regionState=OPENING, regionLocation=9c732c6543ff,43171,1731974958985
2024-11-19T00:09:34,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=bc02cd12a280aa6311e039f4a9aaf2b1, ASSIGN because future has completed
2024-11-19T00:09:34,991 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure bc02cd12a280aa6311e039f4a9aaf2b1, server=9c732c6543ff,43171,1731974958985}]
2024-11-19T00:09:34,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=cab914c3a7324c032510f1518a7f8646, ASSIGN because future has completed
2024-11-19T00:09:34,992 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure cab914c3a7324c032510f1518a7f8646, server=9c732c6543ff,43171,1731974958985}]
2024-11-19T00:09:35,148 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646.
2024-11-19T00:09:35,148 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => cab914c3a7324c032510f1518a7f8646, NAME => 'TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646.', STARTKEY => '', ENDKEY => 'row0062'}
2024-11-19T00:09:35,148 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling cab914c3a7324c032510f1518a7f8646
2024-11-19T00:09:35,148 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:09:35,149 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for cab914c3a7324c032510f1518a7f8646
2024-11-19T00:09:35,149 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for cab914c3a7324c032510f1518a7f8646
2024-11-19T00:09:35,150 INFO  [StoreOpener-cab914c3a7324c032510f1518a7f8646-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region cab914c3a7324c032510f1518a7f8646 
2024-11-19T00:09:35,151 INFO  [StoreOpener-cab914c3a7324c032510f1518a7f8646-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cab914c3a7324c032510f1518a7f8646 columnFamilyName info
2024-11-19T00:09:35,151 DEBUG [StoreOpener-cab914c3a7324c032510f1518a7f8646-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:35,163 DEBUG [StoreOpener-cab914c3a7324c032510f1518a7f8646-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/cab914c3a7324c032510f1518a7f8646/info/768601cfc5ea4e6c8fd7f2e26d010790.5f64dec1279ff8bc85e4392992959c31->hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/768601cfc5ea4e6c8fd7f2e26d010790-bottom
2024-11-19T00:09:35,164 INFO  [StoreOpener-cab914c3a7324c032510f1518a7f8646-1 {}] regionserver.HStore(327): Store=cab914c3a7324c032510f1518a7f8646/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:09:35,164 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for cab914c3a7324c032510f1518a7f8646
2024-11-19T00:09:35,165 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/cab914c3a7324c032510f1518a7f8646
2024-11-19T00:09:35,166 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/cab914c3a7324c032510f1518a7f8646
2024-11-19T00:09:35,167 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for cab914c3a7324c032510f1518a7f8646
2024-11-19T00:09:35,167 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for cab914c3a7324c032510f1518a7f8646
2024-11-19T00:09:35,169 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for cab914c3a7324c032510f1518a7f8646
2024-11-19T00:09:35,171 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened cab914c3a7324c032510f1518a7f8646; next sequenceid=127; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=846930, jitterRate=0.07692797482013702}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-19T00:09:35,171 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cab914c3a7324c032510f1518a7f8646
2024-11-19T00:09:35,172 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for cab914c3a7324c032510f1518a7f8646:
Running coprocessor pre-open hook at 1731974975149Writing region info on filesystem at 1731974975149Initializing all the Stores at 1731974975150 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974975150Cleaning up temporary data from old regions at 1731974975167 (+17 ms)Running coprocessor post-open hooks at 1731974975171 (+4 ms)Region opened successfully at 1731974975172 (+1 ms)
2024-11-19T00:09:35,173 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646., pid=13, masterSystemTime=1731974975144
2024-11-19T00:09:35,174 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store cab914c3a7324c032510f1518a7f8646:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T00:09:35,174 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:09:35,174 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-11-19T00:09:35,174 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646.
2024-11-19T00:09:35,175 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1541): cab914c3a7324c032510f1518a7f8646/info is initiating minor compaction (all files)
2024-11-19T00:09:35,175 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab914c3a7324c032510f1518a7f8646/info in TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646.
2024-11-19T00:09:35,175 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/cab914c3a7324c032510f1518a7f8646/info/768601cfc5ea4e6c8fd7f2e26d010790.5f64dec1279ff8bc85e4392992959c31->hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/768601cfc5ea4e6c8fd7f2e26d010790-bottom] into tmpdir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/cab914c3a7324c032510f1518a7f8646/.tmp, totalSize=73.6 K
2024-11-19T00:09:35,176 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 768601cfc5ea4e6c8fd7f2e26d010790.5f64dec1279ff8bc85e4392992959c31, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731974970092
2024-11-19T00:09:35,181 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646.
2024-11-19T00:09:35,181 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646.
2024-11-19T00:09:35,181 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.
2024-11-19T00:09:35,181 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => bc02cd12a280aa6311e039f4a9aaf2b1, NAME => 'TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.', STARTKEY => 'row0062', ENDKEY => ''}
2024-11-19T00:09:35,181 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:35,182 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:09:35,182 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:35,182 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:35,182 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=cab914c3a7324c032510f1518a7f8646, regionState=OPEN, openSeqNum=127, regionLocation=9c732c6543ff,43171,1731974958985
2024-11-19T00:09:35,183 INFO  [StoreOpener-bc02cd12a280aa6311e039f4a9aaf2b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region bc02cd12a280aa6311e039f4a9aaf2b1 
2024-11-19T00:09:35,185 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-11-19T00:09:35,185 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-11-19T00:09:35,185 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB
2024-11-19T00:09:35,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure cab914c3a7324c032510f1518a7f8646, server=9c732c6543ff,43171,1731974958985 because future has completed
2024-11-19T00:09:35,186 INFO  [StoreOpener-bc02cd12a280aa6311e039f4a9aaf2b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc02cd12a280aa6311e039f4a9aaf2b1 columnFamilyName info
2024-11-19T00:09:35,186 DEBUG [StoreOpener-bc02cd12a280aa6311e039f4a9aaf2b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:09:35,201 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10
2024-11-19T00:09:35,201 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure cab914c3a7324c032510f1518a7f8646, server=9c732c6543ff,43171,1731974958985 in 206 msec
2024-11-19T00:09:35,204 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=cab914c3a7324c032510f1518a7f8646, ASSIGN in 369 msec
2024-11-19T00:09:35,204 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab914c3a7324c032510f1518a7f8646#info#compaction#66 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T00:09:35,205 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/cab914c3a7324c032510f1518a7f8646/.tmp/info/647fdd7f2e4443d7a5d56790c3b2593d is 1080, key is row0001/info:/1731974970092/Put/seqid=0
2024-11-19T00:09:35,212 DEBUG [StoreOpener-bc02cd12a280aa6311e039f4a9aaf2b1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/768601cfc5ea4e6c8fd7f2e26d010790.5f64dec1279ff8bc85e4392992959c31->hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/768601cfc5ea4e6c8fd7f2e26d010790-top
2024-11-19T00:09:35,221 DEBUG [StoreOpener-bc02cd12a280aa6311e039f4a9aaf2b1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-0df66122f0bf4e3a9bcade95ecfb78d1
2024-11-19T00:09:35,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/.tmp/info/c05121941bce45c6bda65957da092846 is 193, key is TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1./info:regioninfo/1731974974987/Put/seqid=0
2024-11-19T00:09:35,228 DEBUG [StoreOpener-bc02cd12a280aa6311e039f4a9aaf2b1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-5341192a871b4fbfae47647135d6345b
2024-11-19T00:09:35,233 DEBUG [StoreOpener-bc02cd12a280aa6311e039f4a9aaf2b1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-9cee2088f42049bdbe5d46dde8af5ef8
2024-11-19T00:09:35,234 INFO  [StoreOpener-bc02cd12a280aa6311e039f4a9aaf2b1-1 {}] regionserver.HStore(327): Store=bc02cd12a280aa6311e039f4a9aaf2b1/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:09:35,234 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:35,234 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:35,236 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:35,236 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:35,237 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:35,239 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:35,241 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened bc02cd12a280aa6311e039f4a9aaf2b1; next sequenceid=127; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=866600, jitterRate=0.10193973779678345}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-19T00:09:35,241 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:35,241 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for bc02cd12a280aa6311e039f4a9aaf2b1:
Running coprocessor pre-open hook at 1731974975182Writing region info on filesystem at 1731974975182Initializing all the Stores at 1731974975183 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974975183Cleaning up temporary data from old regions at 1731974975237 (+54 ms)Running coprocessor post-open hooks at 1731974975241 (+4 ms)Region opened successfully at 1731974975241
2024-11-19T00:09:35,242 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., pid=12, masterSystemTime=1731974975144
2024-11-19T00:09:35,242 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store bc02cd12a280aa6311e039f4a9aaf2b1:info, priority=-2147483648, current under compaction store size is 2
2024-11-19T00:09:35,242 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:09:35,242 DEBUG [RS:0;9c732c6543ff:43171-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking
2024-11-19T00:09:35,245 INFO  [RS:0;9c732c6543ff:43171-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.
2024-11-19T00:09:35,245 DEBUG [RS:0;9c732c6543ff:43171-longCompactions-0 {}] regionserver.HStore(1541): bc02cd12a280aa6311e039f4a9aaf2b1/info is initiating minor compaction (all files)
2024-11-19T00:09:35,245 INFO  [RS:0;9c732c6543ff:43171-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc02cd12a280aa6311e039f4a9aaf2b1/info in TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.
2024-11-19T00:09:35,245 INFO  [RS:0;9c732c6543ff:43171-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/768601cfc5ea4e6c8fd7f2e26d010790.5f64dec1279ff8bc85e4392992959c31->hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/768601cfc5ea4e6c8fd7f2e26d010790-top, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-9cee2088f42049bdbe5d46dde8af5ef8, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-5341192a871b4fbfae47647135d6345b, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-0df66122f0bf4e3a9bcade95ecfb78d1] into tmpdir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp, totalSize=120.8 K
2024-11-19T00:09:35,245 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.
2024-11-19T00:09:35,245 INFO  [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.
2024-11-19T00:09:35,246 DEBUG [RS:0;9c732c6543ff:43171-longCompactions-0 {}] compactions.Compactor(225): Compacting 768601cfc5ea4e6c8fd7f2e26d010790.5f64dec1279ff8bc85e4392992959c31, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731974970092
2024-11-19T00:09:35,246 DEBUG [RS:0;9c732c6543ff:43171-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-9cee2088f42049bdbe5d46dde8af5ef8, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731974974260
2024-11-19T00:09:35,246 DEBUG [RS:0;9c732c6543ff:43171-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-5341192a871b4fbfae47647135d6345b, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731974974291
2024-11-19T00:09:35,247 DEBUG [RS:0;9c732c6543ff:43171-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-0df66122f0bf4e3a9bcade95ecfb78d1, keycount=3, bloomtype=ROW, size=8.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731974974334
2024-11-19T00:09:35,247 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=bc02cd12a280aa6311e039f4a9aaf2b1, regionState=OPEN, openSeqNum=127, regionLocation=9c732c6543ff,43171,1731974958985
2024-11-19T00:09:35,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure bc02cd12a280aa6311e039f4a9aaf2b1, server=9c732c6543ff,43171,1731974958985 because future has completed
2024-11-19T00:09:35,262 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741851_1027 (size=70862)
2024-11-19T00:09:35,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741851_1027 (size=70862)
2024-11-19T00:09:35,272 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40849 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=9c732c6543ff,43171,1731974958985, table=TestLogRolling-testLogRolling, region=bc02cd12a280aa6311e039f4a9aaf2b1. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-19T00:09:35,276 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/cab914c3a7324c032510f1518a7f8646/.tmp/info/647fdd7f2e4443d7a5d56790c3b2593d as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/cab914c3a7324c032510f1518a7f8646/info/647fdd7f2e4443d7a5d56790c3b2593d
2024-11-19T00:09:35,278 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-19T00:09:35,278 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure bc02cd12a280aa6311e039f4a9aaf2b1, server=9c732c6543ff,43171,1731974958985 in 283 msec
2024-11-19T00:09:35,283 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7
2024-11-19T00:09:35,283 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=bc02cd12a280aa6311e039f4a9aaf2b1, ASSIGN in 445 msec
2024-11-19T00:09:35,286 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=5f64dec1279ff8bc85e4392992959c31, daughterA=cab914c3a7324c032510f1518a7f8646, daughterB=bc02cd12a280aa6311e039f4a9aaf2b1 in 905 msec
2024-11-19T00:09:35,290 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in cab914c3a7324c032510f1518a7f8646/info of cab914c3a7324c032510f1518a7f8646 into 647fdd7f2e4443d7a5d56790c3b2593d(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T00:09:35,290 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab914c3a7324c032510f1518a7f8646:

2024-11-19T00:09:35,290 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646., storeName=cab914c3a7324c032510f1518a7f8646/info, priority=15, startTime=1731974975173; duration=0sec
2024-11-19T00:09:35,291 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:09:35,291 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab914c3a7324c032510f1518a7f8646:info
2024-11-19T00:09:35,303 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741852_1028 (size=9847)
2024-11-19T00:09:35,304 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741852_1028 (size=9847)
2024-11-19T00:09:35,305 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/.tmp/info/c05121941bce45c6bda65957da092846
2024-11-19T00:09:35,330 INFO  [RS:0;9c732c6543ff:43171-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc02cd12a280aa6311e039f4a9aaf2b1#info#compaction#68 average throughput is 8.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T00:09:35,331 DEBUG [RS:0;9c732c6543ff:43171-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/3448be3640e94e0181f28781024dfa10 is 1080, key is row0062/info:/1731974972252/Put/seqid=0
2024-11-19T00:09:35,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/.tmp/ns/b0c6bfc8a3d34a26b1cc11db76391cfc is 43, key is default/ns:d/1731974959975/Put/seqid=0
2024-11-19T00:09:35,380 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741854_1030 (size=5153)
2024-11-19T00:09:35,380 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741854_1030 (size=5153)
2024-11-19T00:09:35,380 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/.tmp/ns/b0c6bfc8a3d34a26b1cc11db76391cfc
2024-11-19T00:09:35,382 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741853_1029 (size=43081)
2024-11-19T00:09:35,383 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741853_1029 (size=43081)
2024-11-19T00:09:35,392 DEBUG [RS:0;9c732c6543ff:43171-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/3448be3640e94e0181f28781024dfa10 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/3448be3640e94e0181f28781024dfa10
2024-11-19T00:09:35,403 INFO  [RS:0;9c732c6543ff:43171-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bc02cd12a280aa6311e039f4a9aaf2b1/info of bc02cd12a280aa6311e039f4a9aaf2b1 into 3448be3640e94e0181f28781024dfa10(size=42.1 K), total size for store is 42.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T00:09:35,403 DEBUG [RS:0;9c732c6543ff:43171-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:09:35,403 INFO  [RS:0;9c732c6543ff:43171-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., storeName=bc02cd12a280aa6311e039f4a9aaf2b1/info, priority=12, startTime=1731974975242; duration=0sec
2024-11-19T00:09:35,404 DEBUG [RS:0;9c732c6543ff:43171-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:09:35,405 DEBUG [RS:0;9c732c6543ff:43171-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc02cd12a280aa6311e039f4a9aaf2b1:info
2024-11-19T00:09:35,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/.tmp/table/e42fc35081b946088da39ef6711121a9 is 65, key is TestLogRolling-testLogRolling/table:state/1731974960439/Put/seqid=0
2024-11-19T00:09:35,444 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741855_1031 (size=5340)
2024-11-19T00:09:35,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741855_1031 (size=5340)
2024-11-19T00:09:35,447 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/.tmp/table/e42fc35081b946088da39ef6711121a9
2024-11-19T00:09:35,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/.tmp/info/c05121941bce45c6bda65957da092846 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/info/c05121941bce45c6bda65957da092846
2024-11-19T00:09:35,461 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/info/c05121941bce45c6bda65957da092846, entries=30, sequenceid=17, filesize=9.6 K
2024-11-19T00:09:35,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/.tmp/ns/b0c6bfc8a3d34a26b1cc11db76391cfc as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/ns/b0c6bfc8a3d34a26b1cc11db76391cfc
2024-11-19T00:09:35,471 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/ns/b0c6bfc8a3d34a26b1cc11db76391cfc, entries=2, sequenceid=17, filesize=5.0 K
2024-11-19T00:09:35,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/.tmp/table/e42fc35081b946088da39ef6711121a9 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/table/e42fc35081b946088da39ef6711121a9
2024-11-19T00:09:35,484 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/table/e42fc35081b946088da39ef6711121a9, entries=2, sequenceid=17, filesize=5.2 K
2024-11-19T00:09:35,486 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 301ms, sequenceid=17, compaction requested=false
2024-11-19T00:09:35,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-19T00:09:35,767 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:35,821 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:35,828 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43171 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:51446 deadline: 1731974986340, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31. is not online on 9c732c6543ff,43171,1731974958985
2024-11-19T00:09:36,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31., hostname=9c732c6543ff,43171,1731974958985, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31., hostname=9c732c6543ff,43171,1731974958985, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31. is not online on 9c732c6543ff,43171,1731974958985
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-19T00:09:36,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31., hostname=9c732c6543ff,43171,1731974958985, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31. is not online on 9c732c6543ff,43171,1731974958985
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-19T00:09:36,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1731974960060.5f64dec1279ff8bc85e4392992959c31., hostname=9c732c6543ff,43171,1731974958985, seqNum=2 from cache
2024-11-19T00:09:36,768 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:36,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:36,828 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:37,768 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:37,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:37,829 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:38,769 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:38,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:38,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:39,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:39,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:39,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:40,106 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T00:09:40,107 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,107 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,107 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,107 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,108 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,108 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,108 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,109 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,127 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,127 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,128 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,128 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,128 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,128 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,131 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,131 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,132 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,134 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T00:09:40,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:40,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:40,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:41,771 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:41,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:41,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:42,771 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:42,825 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:42,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:43,772 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:43,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:43,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:44,772 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:44,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:44,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:45,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:45,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:45,833 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., hostname=9c732c6543ff,43171,1731974958985, seqNum=127]
2024-11-19T00:09:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:46,419 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc02cd12a280aa6311e039f4a9aaf2b1 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T00:09:46,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/3b40f7b3e850405aae8fee3c9b8d7c2a is 1080, key is row0097/info:/1731974986410/Put/seqid=0
2024-11-19T00:09:46,430 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741856_1032 (size=12516)
2024-11-19T00:09:46,430 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741856_1032 (size=12516)
2024-11-19T00:09:46,431 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/3b40f7b3e850405aae8fee3c9b8d7c2a
2024-11-19T00:09:46,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/3b40f7b3e850405aae8fee3c9b8d7c2a as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/3b40f7b3e850405aae8fee3c9b8d7c2a
2024-11-19T00:09:46,442 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/3b40f7b3e850405aae8fee3c9b8d7c2a, entries=7, sequenceid=137, filesize=12.2 K
2024-11-19T00:09:46,443 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=15.76 KB/16140 for bc02cd12a280aa6311e039f4a9aaf2b1 in 24ms, sequenceid=137, compaction requested=false
2024-11-19T00:09:46,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:09:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:46,444 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc02cd12a280aa6311e039f4a9aaf2b1 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-11-19T00:09:46,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/68d2e5d343b9415cba537cd681334224 is 1080, key is row0104/info:/1731974986420/Put/seqid=0
2024-11-19T00:09:46,456 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741857_1033 (size=22238)
2024-11-19T00:09:46,457 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741857_1033 (size=22238)
2024-11-19T00:09:46,457 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/68d2e5d343b9415cba537cd681334224
2024-11-19T00:09:46,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/68d2e5d343b9415cba537cd681334224 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/68d2e5d343b9415cba537cd681334224
2024-11-19T00:09:46,466 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/68d2e5d343b9415cba537cd681334224, entries=16, sequenceid=156, filesize=21.7 K
2024-11-19T00:09:46,467 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=9.46 KB/9684 for bc02cd12a280aa6311e039f4a9aaf2b1 in 23ms, sequenceid=156, compaction requested=true
2024-11-19T00:09:46,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:09:46,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc02cd12a280aa6311e039f4a9aaf2b1:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T00:09:46,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:09:46,468 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T00:09:46,469 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 77835 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T00:09:46,469 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1541): bc02cd12a280aa6311e039f4a9aaf2b1/info is initiating minor compaction (all files)
2024-11-19T00:09:46,469 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc02cd12a280aa6311e039f4a9aaf2b1/info in TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.
2024-11-19T00:09:46,469 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/3448be3640e94e0181f28781024dfa10, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/3b40f7b3e850405aae8fee3c9b8d7c2a, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/68d2e5d343b9415cba537cd681334224] into tmpdir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp, totalSize=76.0 K
2024-11-19T00:09:46,469 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3448be3640e94e0181f28781024dfa10, keycount=35, bloomtype=ROW, size=42.1 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731974972252
2024-11-19T00:09:46,469 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b40f7b3e850405aae8fee3c9b8d7c2a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731974986410
2024-11-19T00:09:46,470 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68d2e5d343b9415cba537cd681334224, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731974986420
2024-11-19T00:09:46,480 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc02cd12a280aa6311e039f4a9aaf2b1#info#compaction#73 average throughput is 29.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T00:09:46,480 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/4694b87f4b9e43bc9ca00eabed100452 is 1080, key is row0062/info:/1731974972252/Put/seqid=0
2024-11-19T00:09:46,484 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741858_1034 (size=68045)
2024-11-19T00:09:46,485 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741858_1034 (size=68045)
2024-11-19T00:09:46,490 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/4694b87f4b9e43bc9ca00eabed100452 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/4694b87f4b9e43bc9ca00eabed100452
2024-11-19T00:09:46,495 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc02cd12a280aa6311e039f4a9aaf2b1/info of bc02cd12a280aa6311e039f4a9aaf2b1 into 4694b87f4b9e43bc9ca00eabed100452(size=66.5 K), total size for store is 66.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T00:09:46,495 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:09:46,495 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., storeName=bc02cd12a280aa6311e039f4a9aaf2b1/info, priority=13, startTime=1731974986467; duration=0sec
2024-11-19T00:09:46,495 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:09:46,495 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc02cd12a280aa6311e039f4a9aaf2b1:info
2024-11-19T00:09:46,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:46,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:46,833 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:47,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:47,828 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:47,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:48,461 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc02cd12a280aa6311e039f4a9aaf2b1 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-19T00:09:48,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/36b515496c1d487d8298d7c80f45f6da is 1080, key is row0120/info:/1731974986445/Put/seqid=0
2024-11-19T00:09:48,475 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741859_1035 (size=15750)
2024-11-19T00:09:48,475 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741859_1035 (size=15750)
2024-11-19T00:09:48,476 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/36b515496c1d487d8298d7c80f45f6da
2024-11-19T00:09:48,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/36b515496c1d487d8298d7c80f45f6da as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/36b515496c1d487d8298d7c80f45f6da
2024-11-19T00:09:48,493 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/36b515496c1d487d8298d7c80f45f6da, entries=10, sequenceid=170, filesize=15.4 K
2024-11-19T00:09:48,494 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=14.71 KB/15064 for bc02cd12a280aa6311e039f4a9aaf2b1 in 33ms, sequenceid=170, compaction requested=false
2024-11-19T00:09:48,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:09:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:48,495 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc02cd12a280aa6311e039f4a9aaf2b1 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-11-19T00:09:48,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/0d6e4c3145fc4c759620b75d1c9795e6 is 1080, key is row0130/info:/1731974988463/Put/seqid=0
2024-11-19T00:09:48,519 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741860_1036 (size=21156)
2024-11-19T00:09:48,519 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741860_1036 (size=21156)
2024-11-19T00:09:48,520 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/0d6e4c3145fc4c759620b75d1c9795e6
2024-11-19T00:09:48,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/0d6e4c3145fc4c759620b75d1c9795e6 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/0d6e4c3145fc4c759620b75d1c9795e6
2024-11-19T00:09:48,529 WARN  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43171 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=bc02cd12a280aa6311e039f4a9aaf2b1, server=9c732c6543ff,43171,1731974958985
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-19T00:09:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43171 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:51446 deadline: 1731974998528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=bc02cd12a280aa6311e039f4a9aaf2b1, server=9c732c6543ff,43171,1731974958985
2024-11-19T00:09:48,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., hostname=9c732c6543ff,43171,1731974958985, seqNum=127 , the old value is region=TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., hostname=9c732c6543ff,43171,1731974958985, seqNum=127, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=bc02cd12a280aa6311e039f4a9aaf2b1, server=9c732c6543ff,43171,1731974958985
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-19T00:09:48,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., hostname=9c732c6543ff,43171,1731974958985, seqNum=127 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=bc02cd12a280aa6311e039f4a9aaf2b1, server=9c732c6543ff,43171,1731974958985
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-19T00:09:48,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., hostname=9c732c6543ff,43171,1731974958985, seqNum=127 because the exception is null or not the one we care about
2024-11-19T00:09:48,534 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/0d6e4c3145fc4c759620b75d1c9795e6, entries=15, sequenceid=188, filesize=20.7 K
2024-11-19T00:09:48,535 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=14.71 KB/15064 for bc02cd12a280aa6311e039f4a9aaf2b1 in 40ms, sequenceid=188, compaction requested=true
2024-11-19T00:09:48,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:09:48,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc02cd12a280aa6311e039f4a9aaf2b1:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T00:09:48,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:09:48,536 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T00:09:48,537 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 104951 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T00:09:48,537 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1541): bc02cd12a280aa6311e039f4a9aaf2b1/info is initiating minor compaction (all files)
2024-11-19T00:09:48,537 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc02cd12a280aa6311e039f4a9aaf2b1/info in TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.
2024-11-19T00:09:48,537 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/4694b87f4b9e43bc9ca00eabed100452, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/36b515496c1d487d8298d7c80f45f6da, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/0d6e4c3145fc4c759620b75d1c9795e6] into tmpdir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp, totalSize=102.5 K
2024-11-19T00:09:48,537 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4694b87f4b9e43bc9ca00eabed100452, keycount=58, bloomtype=ROW, size=66.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731974972252
2024-11-19T00:09:48,538 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36b515496c1d487d8298d7c80f45f6da, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731974986445
2024-11-19T00:09:48,538 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d6e4c3145fc4c759620b75d1c9795e6, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731974988463
2024-11-19T00:09:48,551 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc02cd12a280aa6311e039f4a9aaf2b1#info#compaction#76 average throughput is 42.59 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T00:09:48,552 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/cd993a082ad84c0e973839266bcc6d67 is 1080, key is row0062/info:/1731974972252/Put/seqid=0
2024-11-19T00:09:48,573 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741861_1037 (size=95174)
2024-11-19T00:09:48,573 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741861_1037 (size=95174)
2024-11-19T00:09:48,580 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/cd993a082ad84c0e973839266bcc6d67 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/cd993a082ad84c0e973839266bcc6d67
2024-11-19T00:09:48,589 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc02cd12a280aa6311e039f4a9aaf2b1/info of bc02cd12a280aa6311e039f4a9aaf2b1 into cd993a082ad84c0e973839266bcc6d67(size=92.9 K), total size for store is 92.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T00:09:48,589 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:09:48,589 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., storeName=bc02cd12a280aa6311e039f4a9aaf2b1/info, priority=13, startTime=1731974988536; duration=0sec
2024-11-19T00:09:48,589 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:09:48,589 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc02cd12a280aa6311e039f4a9aaf2b1:info
2024-11-19T00:09:48,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:48,829 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:48,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:48,929 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-19T00:09:49,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:49,829 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:49,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:50,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:50,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:50,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:51,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:51,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:51,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:52,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:52,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:52,836 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:53,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:53,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:53,836 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:54,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:54,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:54,837 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:55,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:55,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:55,837 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:56,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:56,833 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:56,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:57,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:57,833 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:57,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:09:58,576 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc02cd12a280aa6311e039f4a9aaf2b1 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-11-19T00:09:58,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/84bd2e8772e148f8ba7e0599ace8cfa4 is 1080, key is row0145/info:/1731974988496/Put/seqid=0
2024-11-19T00:09:58,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741862_1038 (size=21156)
2024-11-19T00:09:58,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741862_1038 (size=21156)
2024-11-19T00:09:58,588 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/84bd2e8772e148f8ba7e0599ace8cfa4
2024-11-19T00:09:58,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/84bd2e8772e148f8ba7e0599ace8cfa4 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/84bd2e8772e148f8ba7e0599ace8cfa4
2024-11-19T00:09:58,599 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/84bd2e8772e148f8ba7e0599ace8cfa4, entries=15, sequenceid=207, filesize=20.7 K
2024-11-19T00:09:58,600 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=1.05 KB/1076 for bc02cd12a280aa6311e039f4a9aaf2b1 in 25ms, sequenceid=207, compaction requested=false
2024-11-19T00:09:58,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:09:58,780 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:58,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:58,839 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:59,780 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:59,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:59,839 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:09:59,992 INFO  [master/9c732c6543ff:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-19T00:09:59,992 INFO  [master/9c732c6543ff:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-19T00:10:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:10:00,590 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc02cd12a280aa6311e039f4a9aaf2b1 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T00:10:00,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/093680918970418f9b8ebd8f1b6de214 is 1080, key is row0160/info:/1731974998577/Put/seqid=0
2024-11-19T00:10:00,601 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741863_1039 (size=12516)
2024-11-19T00:10:00,602 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741863_1039 (size=12516)
2024-11-19T00:10:00,603 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/093680918970418f9b8ebd8f1b6de214
2024-11-19T00:10:00,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/093680918970418f9b8ebd8f1b6de214 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/093680918970418f9b8ebd8f1b6de214
2024-11-19T00:10:00,621 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/093680918970418f9b8ebd8f1b6de214, entries=7, sequenceid=217, filesize=12.2 K
2024-11-19T00:10:00,622 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for bc02cd12a280aa6311e039f4a9aaf2b1 in 32ms, sequenceid=217, compaction requested=true
2024-11-19T00:10:00,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:10:00,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc02cd12a280aa6311e039f4a9aaf2b1:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T00:10:00,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:10:00,622 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T00:10:00,624 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 128846 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T00:10:00,624 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1541): bc02cd12a280aa6311e039f4a9aaf2b1/info is initiating minor compaction (all files)
2024-11-19T00:10:00,624 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc02cd12a280aa6311e039f4a9aaf2b1/info in TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.
2024-11-19T00:10:00,624 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/cd993a082ad84c0e973839266bcc6d67, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/84bd2e8772e148f8ba7e0599ace8cfa4, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/093680918970418f9b8ebd8f1b6de214] into tmpdir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp, totalSize=125.8 K
2024-11-19T00:10:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:10:00,625 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd993a082ad84c0e973839266bcc6d67, keycount=83, bloomtype=ROW, size=92.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731974972252
2024-11-19T00:10:00,625 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc02cd12a280aa6311e039f4a9aaf2b1 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-19T00:10:00,625 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84bd2e8772e148f8ba7e0599ace8cfa4, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731974988496
2024-11-19T00:10:00,626 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 093680918970418f9b8ebd8f1b6de214, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731974998577
2024-11-19T00:10:00,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/f0ee9793f1c0484e894ffadd7b3b671c is 1080, key is row0167/info:/1731975000592/Put/seqid=0
2024-11-19T00:10:00,651 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741864_1040 (size=16828)
2024-11-19T00:10:00,652 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741864_1040 (size=16828)
2024-11-19T00:10:00,653 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/f0ee9793f1c0484e894ffadd7b3b671c
2024-11-19T00:10:00,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/f0ee9793f1c0484e894ffadd7b3b671c as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/f0ee9793f1c0484e894ffadd7b3b671c
2024-11-19T00:10:00,663 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc02cd12a280aa6311e039f4a9aaf2b1#info#compaction#80 average throughput is 35.92 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T00:10:00,664 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/48c3c5f29eb947b0b4c861bee3760ec1 is 1080, key is row0062/info:/1731974972252/Put/seqid=0
2024-11-19T00:10:00,668 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/f0ee9793f1c0484e894ffadd7b3b671c, entries=11, sequenceid=231, filesize=16.4 K
2024-11-19T00:10:00,669 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=15.76 KB/16140 for bc02cd12a280aa6311e039f4a9aaf2b1 in 44ms, sequenceid=231, compaction requested=false
2024-11-19T00:10:00,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:10:00,676 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741865_1041 (size=118996)
2024-11-19T00:10:00,676 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741865_1041 (size=118996)
2024-11-19T00:10:00,687 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/48c3c5f29eb947b0b4c861bee3760ec1 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/48c3c5f29eb947b0b4c861bee3760ec1
2024-11-19T00:10:00,695 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc02cd12a280aa6311e039f4a9aaf2b1/info of bc02cd12a280aa6311e039f4a9aaf2b1 into 48c3c5f29eb947b0b4c861bee3760ec1(size=116.2 K), total size for store is 132.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T00:10:00,695 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:10:00,695 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., storeName=bc02cd12a280aa6311e039f4a9aaf2b1/info, priority=13, startTime=1731975000622; duration=0sec
2024-11-19T00:10:00,696 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:10:00,696 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc02cd12a280aa6311e039f4a9aaf2b1:info
2024-11-19T00:10:00,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:00,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:00,839 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:01,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:01,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:01,840 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:10:02,669 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc02cd12a280aa6311e039f4a9aaf2b1 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-11-19T00:10:02,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/e25c3cfba128459cb05d298b1e95a1b0 is 1080, key is row0178/info:/1731975000626/Put/seqid=0
2024-11-19T00:10:02,692 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741866_1042 (size=22238)
2024-11-19T00:10:02,693 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741866_1042 (size=22238)
2024-11-19T00:10:02,694 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/e25c3cfba128459cb05d298b1e95a1b0
2024-11-19T00:10:02,699 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43171 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=bc02cd12a280aa6311e039f4a9aaf2b1, server=9c732c6543ff,43171,1731974958985
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-19T00:10:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43171 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:51446 deadline: 1731975012699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=bc02cd12a280aa6311e039f4a9aaf2b1, server=9c732c6543ff,43171,1731974958985
2024-11-19T00:10:02,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., hostname=9c732c6543ff,43171,1731974958985, seqNum=127 , the old value is region=TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., hostname=9c732c6543ff,43171,1731974958985, seqNum=127, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=bc02cd12a280aa6311e039f4a9aaf2b1, server=9c732c6543ff,43171,1731974958985
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-19T00:10:02,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., hostname=9c732c6543ff,43171,1731974958985, seqNum=127 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=bc02cd12a280aa6311e039f4a9aaf2b1, server=9c732c6543ff,43171,1731974958985
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-19T00:10:02,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., hostname=9c732c6543ff,43171,1731974958985, seqNum=127 because the exception is null or not the one we care about
2024-11-19T00:10:02,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/e25c3cfba128459cb05d298b1e95a1b0 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/e25c3cfba128459cb05d298b1e95a1b0
2024-11-19T00:10:02,708 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/e25c3cfba128459cb05d298b1e95a1b0, entries=16, sequenceid=251, filesize=21.7 K
2024-11-19T00:10:02,710 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=13.66 KB/13988 for bc02cd12a280aa6311e039f4a9aaf2b1 in 40ms, sequenceid=251, compaction requested=true
2024-11-19T00:10:02,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:10:02,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc02cd12a280aa6311e039f4a9aaf2b1:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T00:10:02,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:10:02,710 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T00:10:02,712 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 158062 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T00:10:02,712 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1541): bc02cd12a280aa6311e039f4a9aaf2b1/info is initiating minor compaction (all files)
2024-11-19T00:10:02,712 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc02cd12a280aa6311e039f4a9aaf2b1/info in TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.
2024-11-19T00:10:02,712 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/48c3c5f29eb947b0b4c861bee3760ec1, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/f0ee9793f1c0484e894ffadd7b3b671c, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/e25c3cfba128459cb05d298b1e95a1b0] into tmpdir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp, totalSize=154.4 K
2024-11-19T00:10:02,712 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48c3c5f29eb947b0b4c861bee3760ec1, keycount=105, bloomtype=ROW, size=116.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731974972252
2024-11-19T00:10:02,713 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0ee9793f1c0484e894ffadd7b3b671c, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731975000592
2024-11-19T00:10:02,713 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting e25c3cfba128459cb05d298b1e95a1b0, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731975000626
2024-11-19T00:10:02,732 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc02cd12a280aa6311e039f4a9aaf2b1#info#compaction#82 average throughput is 67.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T00:10:02,733 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/93fba200b2f74fe7952cf68c3395de89 is 1080, key is row0062/info:/1731974972252/Put/seqid=0
2024-11-19T00:10:02,757 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741867_1043 (size=148409)
2024-11-19T00:10:02,757 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741867_1043 (size=148409)
2024-11-19T00:10:02,767 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/93fba200b2f74fe7952cf68c3395de89 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/93fba200b2f74fe7952cf68c3395de89
2024-11-19T00:10:02,777 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc02cd12a280aa6311e039f4a9aaf2b1/info of bc02cd12a280aa6311e039f4a9aaf2b1 into 93fba200b2f74fe7952cf68c3395de89(size=144.9 K), total size for store is 144.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T00:10:02,777 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:10:02,777 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., storeName=bc02cd12a280aa6311e039f4a9aaf2b1/info, priority=13, startTime=1731975002710; duration=0sec
2024-11-19T00:10:02,777 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:10:02,777 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc02cd12a280aa6311e039f4a9aaf2b1:info
2024-11-19T00:10:02,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:02,836 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:02,840 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:03,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:03,837 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:03,841 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:04,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:04,837 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:04,841 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:04,937 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20340
2024-11-19T00:10:05,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:05,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:05,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:06,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:06,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:06,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:07,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:07,839 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:07,843 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:08,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:08,839 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:08,843 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:09,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:09,840 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:09,843 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:10,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:10,840 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:10,844 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:11,787 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:11,841 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:11,844 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:12,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:10:12,745 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc02cd12a280aa6311e039f4a9aaf2b1 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-19T00:10:12,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/26067809d0004607914f0da2bfdd28ac is 1080, key is row0194/info:/1731975002671/Put/seqid=0
2024-11-19T00:10:12,755 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741868_1044 (size=20089)
2024-11-19T00:10:12,756 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741868_1044 (size=20089)
2024-11-19T00:10:12,756 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/26067809d0004607914f0da2bfdd28ac
2024-11-19T00:10:12,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/26067809d0004607914f0da2bfdd28ac as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/26067809d0004607914f0da2bfdd28ac
2024-11-19T00:10:12,768 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/26067809d0004607914f0da2bfdd28ac, entries=14, sequenceid=269, filesize=19.6 K
2024-11-19T00:10:12,769 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=15.76 KB/16140 for bc02cd12a280aa6311e039f4a9aaf2b1 in 24ms, sequenceid=269, compaction requested=false
2024-11-19T00:10:12,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:10:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:10:12,770 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc02cd12a280aa6311e039f4a9aaf2b1 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-11-19T00:10:12,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/e5fee59d95d64688a2f4195f78a96c4c is 1080, key is row0208/info:/1731975012746/Put/seqid=0
2024-11-19T00:10:12,778 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741869_1045 (size=22254)
2024-11-19T00:10:12,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741869_1045 (size=22254)
2024-11-19T00:10:12,779 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/e5fee59d95d64688a2f4195f78a96c4c
2024-11-19T00:10:12,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/e5fee59d95d64688a2f4195f78a96c4c as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/e5fee59d95d64688a2f4195f78a96c4c
2024-11-19T00:10:12,787 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:12,789 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/e5fee59d95d64688a2f4195f78a96c4c, entries=16, sequenceid=288, filesize=21.7 K
2024-11-19T00:10:12,790 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=1.05 KB/1076 for bc02cd12a280aa6311e039f4a9aaf2b1 in 20ms, sequenceid=288, compaction requested=true
2024-11-19T00:10:12,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:10:12,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc02cd12a280aa6311e039f4a9aaf2b1:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T00:10:12,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:10:12,791 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T00:10:12,791 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 190752 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T00:10:12,792 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1541): bc02cd12a280aa6311e039f4a9aaf2b1/info is initiating minor compaction (all files)
2024-11-19T00:10:12,792 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc02cd12a280aa6311e039f4a9aaf2b1/info in TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.
2024-11-19T00:10:12,792 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/93fba200b2f74fe7952cf68c3395de89, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/26067809d0004607914f0da2bfdd28ac, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/e5fee59d95d64688a2f4195f78a96c4c] into tmpdir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp, totalSize=186.3 K
2024-11-19T00:10:12,792 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93fba200b2f74fe7952cf68c3395de89, keycount=132, bloomtype=ROW, size=144.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731974972252
2024-11-19T00:10:12,792 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26067809d0004607914f0da2bfdd28ac, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731975002671
2024-11-19T00:10:12,793 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5fee59d95d64688a2f4195f78a96c4c, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731975012746
2024-11-19T00:10:12,804 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc02cd12a280aa6311e039f4a9aaf2b1#info#compaction#85 average throughput is 55.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T00:10:12,805 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/b7696e172edb43ea9b35ee0a86c2af93 is 1080, key is row0062/info:/1731974972252/Put/seqid=0
2024-11-19T00:10:12,807 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741870_1046 (size=180886)
2024-11-19T00:10:12,807 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741870_1046 (size=180886)
2024-11-19T00:10:12,812 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/b7696e172edb43ea9b35ee0a86c2af93 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/b7696e172edb43ea9b35ee0a86c2af93
2024-11-19T00:10:12,817 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc02cd12a280aa6311e039f4a9aaf2b1/info of bc02cd12a280aa6311e039f4a9aaf2b1 into b7696e172edb43ea9b35ee0a86c2af93(size=176.6 K), total size for store is 176.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T00:10:12,817 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:10:12,817 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., storeName=bc02cd12a280aa6311e039f4a9aaf2b1/info, priority=13, startTime=1731975012790; duration=0sec
2024-11-19T00:10:12,817 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:10:12,817 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc02cd12a280aa6311e039f4a9aaf2b1:info
2024-11-19T00:10:12,841 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:12,844 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:13,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:13,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:13,845 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:10:14,780 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc02cd12a280aa6311e039f4a9aaf2b1 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T00:10:14,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/c72a5bdba4ca4ff48e7d5898bbf9e862 is 1080, key is row0224/info:/1731975012771/Put/seqid=0
2024-11-19T00:10:14,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:14,791 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741871_1047 (size=12523)
2024-11-19T00:10:14,791 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741871_1047 (size=12523)
2024-11-19T00:10:14,791 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/c72a5bdba4ca4ff48e7d5898bbf9e862
2024-11-19T00:10:14,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/c72a5bdba4ca4ff48e7d5898bbf9e862 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/c72a5bdba4ca4ff48e7d5898bbf9e862
2024-11-19T00:10:14,802 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/c72a5bdba4ca4ff48e7d5898bbf9e862, entries=7, sequenceid=299, filesize=12.2 K
2024-11-19T00:10:14,803 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=15.76 KB/16140 for bc02cd12a280aa6311e039f4a9aaf2b1 in 23ms, sequenceid=299, compaction requested=false
2024-11-19T00:10:14,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:10:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43171 {}] regionserver.HRegion(8855): Flush requested on bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:10:14,805 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc02cd12a280aa6311e039f4a9aaf2b1 1/1 column families, dataSize=17.86 KB heapSize=19.38 KB
2024-11-19T00:10:14,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/2d94c7330aa64efb9b41209a06a3a7b1 is 1080, key is row0231/info:/1731975014781/Put/seqid=0
2024-11-19T00:10:14,813 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741872_1048 (size=23333)
2024-11-19T00:10:14,814 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741872_1048 (size=23333)
2024-11-19T00:10:14,814 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.86 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/2d94c7330aa64efb9b41209a06a3a7b1
2024-11-19T00:10:14,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/2d94c7330aa64efb9b41209a06a3a7b1 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/2d94c7330aa64efb9b41209a06a3a7b1
2024-11-19T00:10:14,824 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/2d94c7330aa64efb9b41209a06a3a7b1, entries=17, sequenceid=319, filesize=22.8 K
2024-11-19T00:10:14,825 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~17.86 KB/18292, heapSize ~19.36 KB/19824, currentSize=9.46 KB/9684 for bc02cd12a280aa6311e039f4a9aaf2b1 in 20ms, sequenceid=319, compaction requested=true
2024-11-19T00:10:14,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:10:14,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc02cd12a280aa6311e039f4a9aaf2b1:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T00:10:14,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:10:14,825 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T00:10:14,826 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 216742 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T00:10:14,826 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1541): bc02cd12a280aa6311e039f4a9aaf2b1/info is initiating minor compaction (all files)
2024-11-19T00:10:14,826 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc02cd12a280aa6311e039f4a9aaf2b1/info in TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.
2024-11-19T00:10:14,827 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/b7696e172edb43ea9b35ee0a86c2af93, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/c72a5bdba4ca4ff48e7d5898bbf9e862, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/2d94c7330aa64efb9b41209a06a3a7b1] into tmpdir=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp, totalSize=211.7 K
2024-11-19T00:10:14,827 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7696e172edb43ea9b35ee0a86c2af93, keycount=162, bloomtype=ROW, size=176.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731974972252
2024-11-19T00:10:14,827 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting c72a5bdba4ca4ff48e7d5898bbf9e862, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731975012771
2024-11-19T00:10:14,827 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d94c7330aa64efb9b41209a06a3a7b1, keycount=17, bloomtype=ROW, size=22.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731975014781
2024-11-19T00:10:14,838 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc02cd12a280aa6311e039f4a9aaf2b1#info#compaction#88 average throughput is 63.62 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T00:10:14,838 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/272033c2b2784366b1a88fe8842079da is 1080, key is row0062/info:/1731974972252/Put/seqid=0
2024-11-19T00:10:14,841 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741873_1049 (size=206961)
2024-11-19T00:10:14,841 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741873_1049 (size=206961)
2024-11-19T00:10:14,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:14,845 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:14,847 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/272033c2b2784366b1a88fe8842079da as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/272033c2b2784366b1a88fe8842079da
2024-11-19T00:10:14,853 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc02cd12a280aa6311e039f4a9aaf2b1/info of bc02cd12a280aa6311e039f4a9aaf2b1 into 272033c2b2784366b1a88fe8842079da(size=202.1 K), total size for store is 202.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T00:10:14,853 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:10:14,853 INFO  [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., storeName=bc02cd12a280aa6311e039f4a9aaf2b1/info, priority=13, startTime=1731975014825; duration=0sec
2024-11-19T00:10:14,853 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T00:10:14,853 DEBUG [RS:0;9c732c6543ff:43171-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc02cd12a280aa6311e039f4a9aaf2b1:info
2024-11-19T00:10:15,789 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:15,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:15,846 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:16,789 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:16,819 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-11-19T00:10:16,819 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C43171%2C1731974958985.1731975016819
2024-11-19T00:10:16,826 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:16,826 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:16,826 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:16,827 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:16,827 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:16,827 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/WALs/9c732c6543ff,43171,1731974958985/9c732c6543ff%2C43171%2C1731974958985.1731974959479 with entries=310, filesize=307.89 KB; new WAL /user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/WALs/9c732c6543ff,43171,1731974958985/9c732c6543ff%2C43171%2C1731974958985.1731975016819
2024-11-19T00:10:16,828 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36629:36629),(127.0.0.1/127.0.0.1:42451:42451)]
2024-11-19T00:10:16,828 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/WALs/9c732c6543ff,43171,1731974958985/9c732c6543ff%2C43171%2C1731974958985.1731974959479 is not closed yet, will try archiving it next time
2024-11-19T00:10:16,829 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741833_1009 (size=315283)
2024-11-19T00:10:16,829 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741833_1009 (size=315283)
2024-11-19T00:10:16,833 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing bc02cd12a280aa6311e039f4a9aaf2b1 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-11-19T00:10:16,841 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/aa161c6cda424ac094d393b324afee71 is 1080, key is row0248/info:/1731975014806/Put/seqid=0
2024-11-19T00:10:16,843 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:16,846 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741875_1051 (size=14681)
2024-11-19T00:10:16,846 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741875_1051 (size=14681)
2024-11-19T00:10:16,846 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:16,847 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/aa161c6cda424ac094d393b324afee71
2024-11-19T00:10:16,852 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/.tmp/info/aa161c6cda424ac094d393b324afee71 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/aa161c6cda424ac094d393b324afee71
2024-11-19T00:10:16,858 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/aa161c6cda424ac094d393b324afee71, entries=9, sequenceid=332, filesize=14.3 K
2024-11-19T00:10:16,859 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=0 B/0 for bc02cd12a280aa6311e039f4a9aaf2b1 in 26ms, sequenceid=332, compaction requested=false
2024-11-19T00:10:16,859 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bc02cd12a280aa6311e039f4a9aaf2b1:

2024-11-19T00:10:16,859 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for cab914c3a7324c032510f1518a7f8646:

2024-11-19T00:10:16,860 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB
2024-11-19T00:10:16,864 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/.tmp/info/b7b6cd4c102e4ea1835a9a80c7f4a720 is 193, key is TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1./info:regioninfo/1731974975247/Put/seqid=0
2024-11-19T00:10:16,869 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741876_1052 (size=6223)
2024-11-19T00:10:16,870 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741876_1052 (size=6223)
2024-11-19T00:10:16,870 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/.tmp/info/b7b6cd4c102e4ea1835a9a80c7f4a720
2024-11-19T00:10:16,876 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/.tmp/info/b7b6cd4c102e4ea1835a9a80c7f4a720 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/info/b7b6cd4c102e4ea1835a9a80c7f4a720
2024-11-19T00:10:16,881 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/info/b7b6cd4c102e4ea1835a9a80c7f4a720, entries=5, sequenceid=21, filesize=6.1 K
2024-11-19T00:10:16,882 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 23ms, sequenceid=21, compaction requested=false
2024-11-19T00:10:16,882 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-19T00:10:16,882 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C43171%2C1731974958985.1731975016882
2024-11-19T00:10:16,887 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:16,887 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:16,887 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:16,888 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:16,888 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:16,888 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/WALs/9c732c6543ff,43171,1731974958985/9c732c6543ff%2C43171%2C1731974958985.1731975016819 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/WALs/9c732c6543ff,43171,1731974958985/9c732c6543ff%2C43171%2C1731974958985.1731975016882
2024-11-19T00:10:16,889 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741874_1050 (size=731)
2024-11-19T00:10:16,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741874_1050 (size=731)
2024-11-19T00:10:16,894 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/WALs/9c732c6543ff,43171,1731974958985/9c732c6543ff%2C43171%2C1731974958985.1731974959479 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/oldWALs/9c732c6543ff%2C43171%2C1731974958985.1731974959479
2024-11-19T00:10:16,894 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42451:42451),(127.0.0.1/127.0.0.1:36629:36629)]
2024-11-19T00:10:16,894 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-11-19T00:10:16,895 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-19T00:10:16,895 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T00:10:16,895 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:10:16,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:10:16,895 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/WALs/9c732c6543ff,43171,1731974958985/9c732c6543ff%2C43171%2C1731974958985.1731975016819 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/oldWALs/9c732c6543ff%2C43171%2C1731974958985.1731975016819
2024-11-19T00:10:16,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:10:16,895 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-19T00:10:16,895 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-19T00:10:16,895 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2142555489, stopped=false
2024-11-19T00:10:16,895 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9c732c6543ff,40849,1731974958944
2024-11-19T00:10:16,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T00:10:16,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T00:10:16,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:16,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:16,897 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T00:10:16,898 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T00:10:16,898 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:10:16,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:10:16,898 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9c732c6543ff,43171,1731974958985' *****
2024-11-19T00:10:16,898 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:10:16,898 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-19T00:10:16,898 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-19T00:10:16,899 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-19T00:10:16,899 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:10:16,899 INFO  [RS:0;9c732c6543ff:43171 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-19T00:10:16,899 INFO  [RS:0;9c732c6543ff:43171 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-19T00:10:16,899 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer(3091): Received CLOSE for bc02cd12a280aa6311e039f4a9aaf2b1
2024-11-19T00:10:16,899 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer(3091): Received CLOSE for cab914c3a7324c032510f1518a7f8646
2024-11-19T00:10:16,899 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer(959): stopping server 9c732c6543ff,43171,1731974958985
2024-11-19T00:10:16,899 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T00:10:16,899 INFO  [RS:0;9c732c6543ff:43171 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9c732c6543ff:43171.
2024-11-19T00:10:16,899 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bc02cd12a280aa6311e039f4a9aaf2b1, disabling compactions & flushes
2024-11-19T00:10:16,899 DEBUG [RS:0;9c732c6543ff:43171 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:10:16,899 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.
2024-11-19T00:10:16,899 DEBUG [RS:0;9c732c6543ff:43171 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:10:16,899 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.
2024-11-19T00:10:16,899 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1. after waiting 0 ms
2024-11-19T00:10:16,899 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.
2024-11-19T00:10:16,899 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-19T00:10:16,899 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-19T00:10:16,899 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-19T00:10:16,899 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-19T00:10:16,901 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-11-19T00:10:16,901 DEBUG [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer(1325): Online Regions={bc02cd12a280aa6311e039f4a9aaf2b1=TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1., cab914c3a7324c032510f1518a7f8646=TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646., 1588230740=hbase:meta,,1.1588230740}
2024-11-19T00:10:16,901 DEBUG [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, bc02cd12a280aa6311e039f4a9aaf2b1, cab914c3a7324c032510f1518a7f8646
2024-11-19T00:10:16,901 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T00:10:16,901 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T00:10:16,901 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T00:10:16,901 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T00:10:16,901 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T00:10:16,901 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/768601cfc5ea4e6c8fd7f2e26d010790.5f64dec1279ff8bc85e4392992959c31->hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/768601cfc5ea4e6c8fd7f2e26d010790-top, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-9cee2088f42049bdbe5d46dde8af5ef8, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-5341192a871b4fbfae47647135d6345b, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/3448be3640e94e0181f28781024dfa10, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-0df66122f0bf4e3a9bcade95ecfb78d1, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/3b40f7b3e850405aae8fee3c9b8d7c2a, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/4694b87f4b9e43bc9ca00eabed100452, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/68d2e5d343b9415cba537cd681334224, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/36b515496c1d487d8298d7c80f45f6da, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/cd993a082ad84c0e973839266bcc6d67, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/0d6e4c3145fc4c759620b75d1c9795e6, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/84bd2e8772e148f8ba7e0599ace8cfa4, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/48c3c5f29eb947b0b4c861bee3760ec1, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/093680918970418f9b8ebd8f1b6de214, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/f0ee9793f1c0484e894ffadd7b3b671c, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/93fba200b2f74fe7952cf68c3395de89, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/e25c3cfba128459cb05d298b1e95a1b0, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/26067809d0004607914f0da2bfdd28ac, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/b7696e172edb43ea9b35ee0a86c2af93, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/e5fee59d95d64688a2f4195f78a96c4c, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/c72a5bdba4ca4ff48e7d5898bbf9e862, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/2d94c7330aa64efb9b41209a06a3a7b1] to archive
2024-11-19T00:10:16,902 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-19T00:10:16,904 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/768601cfc5ea4e6c8fd7f2e26d010790.5f64dec1279ff8bc85e4392992959c31 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/768601cfc5ea4e6c8fd7f2e26d010790.5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:10:16,905 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-9cee2088f42049bdbe5d46dde8af5ef8 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-9cee2088f42049bdbe5d46dde8af5ef8
2024-11-19T00:10:16,906 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-11-19T00:10:16,907 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T00:10:16,907 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-5341192a871b4fbfae47647135d6345b to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-5341192a871b4fbfae47647135d6345b
2024-11-19T00:10:16,907 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T00:10:16,907 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731975016901Running coprocessor pre-close hooks at 1731975016901Disabling compacts and flushes for region at 1731975016901Disabling writes for close at 1731975016901Writing region close event to WAL at 1731975016903 (+2 ms)Running coprocessor post-close hooks at 1731975016907 (+4 ms)Closed at 1731975016907
2024-11-19T00:10:16,907 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-19T00:10:16,908 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/3448be3640e94e0181f28781024dfa10 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/3448be3640e94e0181f28781024dfa10
2024-11-19T00:10:16,909 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-0df66122f0bf4e3a9bcade95ecfb78d1 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/TestLogRolling-testLogRolling=5f64dec1279ff8bc85e4392992959c31-0df66122f0bf4e3a9bcade95ecfb78d1
2024-11-19T00:10:16,910 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/3b40f7b3e850405aae8fee3c9b8d7c2a to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/3b40f7b3e850405aae8fee3c9b8d7c2a
2024-11-19T00:10:16,911 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/4694b87f4b9e43bc9ca00eabed100452 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/4694b87f4b9e43bc9ca00eabed100452
2024-11-19T00:10:16,912 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/68d2e5d343b9415cba537cd681334224 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/68d2e5d343b9415cba537cd681334224
2024-11-19T00:10:16,914 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/36b515496c1d487d8298d7c80f45f6da to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/36b515496c1d487d8298d7c80f45f6da
2024-11-19T00:10:16,915 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/cd993a082ad84c0e973839266bcc6d67 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/cd993a082ad84c0e973839266bcc6d67
2024-11-19T00:10:16,916 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/0d6e4c3145fc4c759620b75d1c9795e6 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/0d6e4c3145fc4c759620b75d1c9795e6
2024-11-19T00:10:16,918 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/84bd2e8772e148f8ba7e0599ace8cfa4 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/84bd2e8772e148f8ba7e0599ace8cfa4
2024-11-19T00:10:16,919 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/48c3c5f29eb947b0b4c861bee3760ec1 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/48c3c5f29eb947b0b4c861bee3760ec1
2024-11-19T00:10:16,920 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/093680918970418f9b8ebd8f1b6de214 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/093680918970418f9b8ebd8f1b6de214
2024-11-19T00:10:16,921 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/f0ee9793f1c0484e894ffadd7b3b671c to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/f0ee9793f1c0484e894ffadd7b3b671c
2024-11-19T00:10:16,922 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/93fba200b2f74fe7952cf68c3395de89 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/93fba200b2f74fe7952cf68c3395de89
2024-11-19T00:10:16,924 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/e25c3cfba128459cb05d298b1e95a1b0 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/e25c3cfba128459cb05d298b1e95a1b0
2024-11-19T00:10:16,925 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/26067809d0004607914f0da2bfdd28ac to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/26067809d0004607914f0da2bfdd28ac
2024-11-19T00:10:16,926 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/b7696e172edb43ea9b35ee0a86c2af93 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/b7696e172edb43ea9b35ee0a86c2af93
2024-11-19T00:10:16,928 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/e5fee59d95d64688a2f4195f78a96c4c to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/e5fee59d95d64688a2f4195f78a96c4c
2024-11-19T00:10:16,929 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/c72a5bdba4ca4ff48e7d5898bbf9e862 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/c72a5bdba4ca4ff48e7d5898bbf9e862
2024-11-19T00:10:16,930 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/2d94c7330aa64efb9b41209a06a3a7b1 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/info/2d94c7330aa64efb9b41209a06a3a7b1
2024-11-19T00:10:16,930 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=9c732c6543ff:40849 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-19T00:10:16,930 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [3448be3640e94e0181f28781024dfa10=43081, 3b40f7b3e850405aae8fee3c9b8d7c2a=12516, 4694b87f4b9e43bc9ca00eabed100452=68045, 68d2e5d343b9415cba537cd681334224=22238, 36b515496c1d487d8298d7c80f45f6da=15750, cd993a082ad84c0e973839266bcc6d67=95174, 0d6e4c3145fc4c759620b75d1c9795e6=21156, 84bd2e8772e148f8ba7e0599ace8cfa4=21156, 48c3c5f29eb947b0b4c861bee3760ec1=118996, 093680918970418f9b8ebd8f1b6de214=12516, f0ee9793f1c0484e894ffadd7b3b671c=16828, 93fba200b2f74fe7952cf68c3395de89=148409, e25c3cfba128459cb05d298b1e95a1b0=22238, 26067809d0004607914f0da2bfdd28ac=20089, b7696e172edb43ea9b35ee0a86c2af93=180886, e5fee59d95d64688a2f4195f78a96c4c=22254, c72a5bdba4ca4ff48e7d5898bbf9e862=12523, 2d94c7330aa64efb9b41209a06a3a7b1=23333]
2024-11-19T00:10:16,937 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/bc02cd12a280aa6311e039f4a9aaf2b1/recovered.edits/335.seqid, newMaxSeqId=335, maxSeqId=126
2024-11-19T00:10:16,937 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.
2024-11-19T00:10:16,937 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bc02cd12a280aa6311e039f4a9aaf2b1:
Waiting for close lock at 1731975016899Running coprocessor pre-close hooks at 1731975016899Disabling compacts and flushes for region at 1731975016899Disabling writes for close at 1731975016899Writing region close event to WAL at 1731975016931 (+32 ms)Running coprocessor post-close hooks at 1731975016937 (+6 ms)Closed at 1731975016937
2024-11-19T00:10:16,937 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1731974974377.bc02cd12a280aa6311e039f4a9aaf2b1.
2024-11-19T00:10:16,938 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cab914c3a7324c032510f1518a7f8646, disabling compactions & flushes
2024-11-19T00:10:16,938 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646.
2024-11-19T00:10:16,938 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646.
2024-11-19T00:10:16,938 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646. after waiting 0 ms
2024-11-19T00:10:16,938 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646.
2024-11-19T00:10:16,938 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/cab914c3a7324c032510f1518a7f8646/info/768601cfc5ea4e6c8fd7f2e26d010790.5f64dec1279ff8bc85e4392992959c31->hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/5f64dec1279ff8bc85e4392992959c31/info/768601cfc5ea4e6c8fd7f2e26d010790-bottom] to archive
2024-11-19T00:10:16,939 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-19T00:10:16,940 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/cab914c3a7324c032510f1518a7f8646/info/768601cfc5ea4e6c8fd7f2e26d010790.5f64dec1279ff8bc85e4392992959c31 to hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/archive/data/default/TestLogRolling-testLogRolling/cab914c3a7324c032510f1518a7f8646/info/768601cfc5ea4e6c8fd7f2e26d010790.5f64dec1279ff8bc85e4392992959c31
2024-11-19T00:10:16,940 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-11-19T00:10:16,943 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/data/default/TestLogRolling-testLogRolling/cab914c3a7324c032510f1518a7f8646/recovered.edits/131.seqid, newMaxSeqId=131, maxSeqId=126
2024-11-19T00:10:16,944 INFO  [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646.
2024-11-19T00:10:16,944 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cab914c3a7324c032510f1518a7f8646:
Waiting for close lock at 1731975016938Running coprocessor pre-close hooks at 1731975016938Disabling compacts and flushes for region at 1731975016938Disabling writes for close at 1731975016938Writing region close event to WAL at 1731975016941 (+3 ms)Running coprocessor post-close hooks at 1731975016944 (+3 ms)Closed at 1731975016944
2024-11-19T00:10:16,944 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1731974974377.cab914c3a7324c032510f1518a7f8646.
2024-11-19T00:10:17,101 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer(976): stopping server 9c732c6543ff,43171,1731974958985; all regions closed.
2024-11-19T00:10:17,102 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:17,102 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:17,102 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:17,102 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:17,102 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:17,104 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741834_1010 (size=8107)
2024-11-19T00:10:17,105 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741834_1010 (size=8107)
2024-11-19T00:10:17,107 DEBUG [RS:0;9c732c6543ff:43171 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/oldWALs
2024-11-19T00:10:17,107 INFO  [RS:0;9c732c6543ff:43171 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9c732c6543ff%2C43171%2C1731974958985.meta:.meta(num 1731974959922)
2024-11-19T00:10:17,107 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:17,107 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:17,107 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:17,108 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:17,108 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:17,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741877_1053 (size=780)
2024-11-19T00:10:17,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741877_1053 (size=780)
2024-11-19T00:10:17,111 DEBUG [RS:0;9c732c6543ff:43171 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/oldWALs
2024-11-19T00:10:17,111 INFO  [RS:0;9c732c6543ff:43171 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9c732c6543ff%2C43171%2C1731974958985:(num 1731975016882)
2024-11-19T00:10:17,111 DEBUG [RS:0;9c732c6543ff:43171 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:10:17,111 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T00:10:17,111 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T00:10:17,112 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.ChoreService(370): Chore service for: regionserver/9c732c6543ff:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-19T00:10:17,112 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T00:10:17,112 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T00:10:17,112 INFO  [RS:0;9c732c6543ff:43171 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43171
2024-11-19T00:10:17,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T00:10:17,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9c732c6543ff,43171,1731974958985
2024-11-19T00:10:17,114 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T00:10:17,115 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9c732c6543ff,43171,1731974958985]
2024-11-19T00:10:17,117 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9c732c6543ff,43171,1731974958985 already deleted, retry=false
2024-11-19T00:10:17,117 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9c732c6543ff,43171,1731974958985 expired; onlineServers=0
2024-11-19T00:10:17,117 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9c732c6543ff,40849,1731974958944' *****
2024-11-19T00:10:17,117 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-19T00:10:17,117 INFO  [M:0;9c732c6543ff:40849 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T00:10:17,117 INFO  [M:0;9c732c6543ff:40849 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T00:10:17,117 DEBUG [M:0;9c732c6543ff:40849 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-19T00:10:17,117 DEBUG [M:0;9c732c6543ff:40849 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-19T00:10:17,117 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-19T00:10:17,117 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974959205 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974959205,5,FailOnTimeoutGroup]
2024-11-19T00:10:17,117 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974959201 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974959201,5,FailOnTimeoutGroup]
2024-11-19T00:10:17,117 INFO  [M:0;9c732c6543ff:40849 {}] hbase.ChoreService(370): Chore service for: master/9c732c6543ff:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-19T00:10:17,117 INFO  [M:0;9c732c6543ff:40849 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T00:10:17,117 DEBUG [M:0;9c732c6543ff:40849 {}] master.HMaster(1795): Stopping service threads
2024-11-19T00:10:17,117 INFO  [M:0;9c732c6543ff:40849 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-19T00:10:17,117 INFO  [M:0;9c732c6543ff:40849 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T00:10:17,118 INFO  [M:0;9c732c6543ff:40849 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-19T00:10:17,118 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-19T00:10:17,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-19T00:10:17,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:17,119 DEBUG [M:0;9c732c6543ff:40849 {}] zookeeper.ZKUtil(347): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-19T00:10:17,119 WARN  [M:0;9c732c6543ff:40849 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-19T00:10:17,119 INFO  [M:0;9c732c6543ff:40849 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/.lastflushedseqids
2024-11-19T00:10:17,124 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741878_1054 (size=228)
2024-11-19T00:10:17,124 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741878_1054 (size=228)
2024-11-19T00:10:17,124 INFO  [M:0;9c732c6543ff:40849 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-19T00:10:17,125 INFO  [M:0;9c732c6543ff:40849 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-19T00:10:17,125 DEBUG [M:0;9c732c6543ff:40849 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T00:10:17,125 INFO  [M:0;9c732c6543ff:40849 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:10:17,125 DEBUG [M:0;9c732c6543ff:40849 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:10:17,125 DEBUG [M:0;9c732c6543ff:40849 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T00:10:17,125 DEBUG [M:0;9c732c6543ff:40849 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:10:17,125 INFO  [M:0;9c732c6543ff:40849 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.42 KB heapSize=63.35 KB
2024-11-19T00:10:17,141 DEBUG [M:0;9c732c6543ff:40849 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bcf65831bf9747428cade4aff2f34483 is 82, key is hbase:meta,,1/info:regioninfo/1731974959956/Put/seqid=0
2024-11-19T00:10:17,145 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741879_1055 (size=5672)
2024-11-19T00:10:17,145 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741879_1055 (size=5672)
2024-11-19T00:10:17,146 INFO  [M:0;9c732c6543ff:40849 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bcf65831bf9747428cade4aff2f34483
2024-11-19T00:10:17,165 DEBUG [M:0;9c732c6543ff:40849 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2f49c3d3c11948dd8fbb1d6a6d5f2409 is 750, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731974960444/Put/seqid=0
2024-11-19T00:10:17,169 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741880_1056 (size=7090)
2024-11-19T00:10:17,169 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741880_1056 (size=7090)
2024-11-19T00:10:17,170 INFO  [M:0;9c732c6543ff:40849 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.81 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2f49c3d3c11948dd8fbb1d6a6d5f2409
2024-11-19T00:10:17,174 INFO  [M:0;9c732c6543ff:40849 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2f49c3d3c11948dd8fbb1d6a6d5f2409
2024-11-19T00:10:17,189 DEBUG [M:0;9c732c6543ff:40849 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cd6b3114fd704b64b2515d65ea34c113 is 69, key is 9c732c6543ff,43171,1731974958985/rs:state/1731974959295/Put/seqid=0
2024-11-19T00:10:17,193 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741881_1057 (size=5156)
2024-11-19T00:10:17,194 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741881_1057 (size=5156)
2024-11-19T00:10:17,194 INFO  [M:0;9c732c6543ff:40849 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cd6b3114fd704b64b2515d65ea34c113
2024-11-19T00:10:17,212 DEBUG [M:0;9c732c6543ff:40849 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/513d999067bc49658f573b016afcac40 is 52, key is load_balancer_on/state:d/1731974960056/Put/seqid=0
2024-11-19T00:10:17,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:10:17,216 INFO  [RS:0;9c732c6543ff:43171 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T00:10:17,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43171-0x101085383fd0001, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:10:17,216 INFO  [RS:0;9c732c6543ff:43171 {}] regionserver.HRegionServer(1031): Exiting; stopping=9c732c6543ff,43171,1731974958985; zookeeper connection closed.
2024-11-19T00:10:17,216 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@75bd9ed9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@75bd9ed9
2024-11-19T00:10:17,216 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-19T00:10:17,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741882_1058 (size=5056)
2024-11-19T00:10:17,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741882_1058 (size=5056)
2024-11-19T00:10:17,217 INFO  [M:0;9c732c6543ff:40849 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/513d999067bc49658f573b016afcac40
2024-11-19T00:10:17,222 DEBUG [M:0;9c732c6543ff:40849 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bcf65831bf9747428cade4aff2f34483 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bcf65831bf9747428cade4aff2f34483
2024-11-19T00:10:17,225 INFO  [M:0;9c732c6543ff:40849 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bcf65831bf9747428cade4aff2f34483, entries=8, sequenceid=125, filesize=5.5 K
2024-11-19T00:10:17,226 DEBUG [M:0;9c732c6543ff:40849 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2f49c3d3c11948dd8fbb1d6a6d5f2409 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2f49c3d3c11948dd8fbb1d6a6d5f2409
2024-11-19T00:10:17,230 INFO  [M:0;9c732c6543ff:40849 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2f49c3d3c11948dd8fbb1d6a6d5f2409
2024-11-19T00:10:17,230 INFO  [M:0;9c732c6543ff:40849 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2f49c3d3c11948dd8fbb1d6a6d5f2409, entries=13, sequenceid=125, filesize=6.9 K
2024-11-19T00:10:17,230 DEBUG [M:0;9c732c6543ff:40849 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cd6b3114fd704b64b2515d65ea34c113 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cd6b3114fd704b64b2515d65ea34c113
2024-11-19T00:10:17,234 INFO  [M:0;9c732c6543ff:40849 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cd6b3114fd704b64b2515d65ea34c113, entries=1, sequenceid=125, filesize=5.0 K
2024-11-19T00:10:17,235 DEBUG [M:0;9c732c6543ff:40849 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/513d999067bc49658f573b016afcac40 as hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/513d999067bc49658f573b016afcac40
2024-11-19T00:10:17,239 INFO  [M:0;9c732c6543ff:40849 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40297/user/jenkins/test-data/488cb02f-c582-e3a8-6951-4acf4eabf2b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/513d999067bc49658f573b016afcac40, entries=1, sequenceid=125, filesize=4.9 K
2024-11-19T00:10:17,240 INFO  [M:0;9c732c6543ff:40849 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.42 KB/52651, heapSize ~63.29 KB/64808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 115ms, sequenceid=125, compaction requested=false
2024-11-19T00:10:17,241 INFO  [M:0;9c732c6543ff:40849 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:10:17,241 DEBUG [M:0;9c732c6543ff:40849 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731975017125Disabling compacts and flushes for region at 1731975017125Disabling writes for close at 1731975017125Obtaining lock to block concurrent updates at 1731975017125Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731975017125Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52651, getHeapSize=64808, getOffHeapSize=0, getCellsCount=148 at 1731975017125Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731975017126 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731975017126Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731975017141 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731975017141Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731975017150 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731975017165 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731975017165Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731975017174 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731975017188 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731975017188Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731975017198 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731975017212 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731975017212Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3885f811: reopening flushed file at 1731975017221 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6efba61: reopening flushed file at 1731975017225 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e4039ca: reopening flushed file at 1731975017230 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58c3b7c5: reopening flushed file at 1731975017234 (+4 ms)Finished flush of dataSize ~51.42 KB/52651, heapSize ~63.29 KB/64808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 115ms, sequenceid=125, compaction requested=false at 1731975017240 (+6 ms)Writing region close event to WAL at 1731975017241 (+1 ms)Closed at 1731975017241
2024-11-19T00:10:17,241 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:17,242 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:17,242 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:17,242 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:17,242 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:17,243 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33047 is added to blk_1073741830_1006 (size=61320)
2024-11-19T00:10:17,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39605 is added to blk_1073741830_1006 (size=61320)
2024-11-19T00:10:17,244 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T00:10:17,244 INFO  [M:0;9c732c6543ff:40849 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-19T00:10:17,244 INFO  [M:0;9c732c6543ff:40849 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40849
2024-11-19T00:10:17,244 INFO  [M:0;9c732c6543ff:40849 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T00:10:17,336 INFO  [regionserver/9c732c6543ff:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T00:10:17,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:10:17,346 INFO  [M:0;9c732c6543ff:40849 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T00:10:17,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40849-0x101085383fd0000, quorum=127.0.0.1:49738, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:10:17,349 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@719b1e37{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:10:17,349 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34accf12{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:10:17,349 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:10:17,349 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3868302b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:10:17,349 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@412b5320{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/hadoop.log.dir/,STOPPED}
2024-11-19T00:10:17,351 WARN  [BP-344767392-172.17.0.2-1731974958292 heartbeating to localhost/127.0.0.1:40297 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:10:17,351 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:10:17,351 WARN  [BP-344767392-172.17.0.2-1731974958292 heartbeating to localhost/127.0.0.1:40297 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-344767392-172.17.0.2-1731974958292 (Datanode Uuid 40c6372d-cffd-48b8-93fc-1738c3405346) service to localhost/127.0.0.1:40297
2024-11-19T00:10:17,351 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:10:17,352 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/cluster_4aa9ef7b-a731-47c8-baac-55880e8d45de/data/data3/current/BP-344767392-172.17.0.2-1731974958292 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:10:17,352 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/cluster_4aa9ef7b-a731-47c8-baac-55880e8d45de/data/data4/current/BP-344767392-172.17.0.2-1731974958292 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:10:17,352 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:10:17,354 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7305dd28{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:10:17,355 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f8818bb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:10:17,355 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:10:17,355 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@516e643a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:10:17,355 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75d9b484{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/hadoop.log.dir/,STOPPED}
2024-11-19T00:10:17,356 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:10:17,356 WARN  [BP-344767392-172.17.0.2-1731974958292 heartbeating to localhost/127.0.0.1:40297 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:10:17,356 WARN  [BP-344767392-172.17.0.2-1731974958292 heartbeating to localhost/127.0.0.1:40297 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-344767392-172.17.0.2-1731974958292 (Datanode Uuid 439bf844-b551-4b29-ae83-aa33a47a0a86) service to localhost/127.0.0.1:40297
2024-11-19T00:10:17,356 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:10:17,357 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/cluster_4aa9ef7b-a731-47c8-baac-55880e8d45de/data/data1/current/BP-344767392-172.17.0.2-1731974958292 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:10:17,357 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/cluster_4aa9ef7b-a731-47c8-baac-55880e8d45de/data/data2/current/BP-344767392-172.17.0.2-1731974958292 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:10:17,357 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:10:17,363 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@e4041f6{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T00:10:17,364 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@c5b7d7a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:10:17,364 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:10:17,364 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@51e0317a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:10:17,364 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@540b6b2e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/hadoop.log.dir/,STOPPED}
2024-11-19T00:10:17,371 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-19T00:10:17,398 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-19T00:10:17,410 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=230 (was 206)
Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40297 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40297
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:40297
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:40297
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40297
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40297 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40297 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40297
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=506 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=245 (was 270), ProcessCount=11 (was 11), AvailableMemoryMB=3545 (was 3661)
2024-11-19T00:10:17,418 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=230, OpenFileDescriptor=506, MaxFileDescriptor=1048576, SystemLoadAverage=245, ProcessCount=11, AvailableMemoryMB=3545
2024-11-19T00:10:17,418 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-19T00:10:17,418 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/hadoop.log.dir so I do NOT create it in target/test-data/05df7a32-a760-2ba2-31df-8317717ac230
2024-11-19T00:10:17,419 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/417ac65e-d831-562f-c917-198a2eb11073/hadoop.tmp.dir so I do NOT create it in target/test-data/05df7a32-a760-2ba2-31df-8317717ac230
2024-11-19T00:10:17,419 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/cluster_62390c19-61af-6e5f-e735-f904e939586e, deleteOnExit=true
2024-11-19T00:10:17,419 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-19T00:10:17,419 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/test.cache.data in system properties and HBase conf
2024-11-19T00:10:17,419 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/hadoop.tmp.dir in system properties and HBase conf
2024-11-19T00:10:17,419 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/hadoop.log.dir in system properties and HBase conf
2024-11-19T00:10:17,419 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-19T00:10:17,419 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-19T00:10:17,419 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-19T00:10:17,419 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-19T00:10:17,419 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-19T00:10:17,419 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-19T00:10:17,419 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-19T00:10:17,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T00:10:17,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-19T00:10:17,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-19T00:10:17,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T00:10:17,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T00:10:17,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-19T00:10:17,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/nfs.dump.dir in system properties and HBase conf
2024-11-19T00:10:17,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/java.io.tmpdir in system properties and HBase conf
2024-11-19T00:10:17,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T00:10:17,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-19T00:10:17,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-19T00:10:17,433 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T00:10:17,489 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:10:17,492 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:10:17,497 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:10:17,497 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:10:17,497 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T00:10:17,498 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:10:17,498 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6ce66601{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:10:17,499 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5fab3ee3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:10:17,611 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6486a7e6{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/java.io.tmpdir/jetty-localhost-39599-hadoop-hdfs-3_4_1-tests_jar-_-any-3736976375526599899/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T00:10:17,612 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34852128{HTTP/1.1, (http/1.1)}{localhost:39599}
2024-11-19T00:10:17,612 INFO  [Time-limited test {}] server.Server(415): Started @296619ms
2024-11-19T00:10:17,625 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T00:10:17,672 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:10:17,675 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:10:17,675 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:10:17,675 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:10:17,675 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T00:10:17,676 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7d6e4d22{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:10:17,676 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@259cffcc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:10:17,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:17,791 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7e09407e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/java.io.tmpdir/jetty-localhost-40821-hadoop-hdfs-3_4_1-tests_jar-_-any-1435765062529708650/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:10:17,791 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@61266769{HTTP/1.1, (http/1.1)}{localhost:40821}
2024-11-19T00:10:17,791 INFO  [Time-limited test {}] server.Server(415): Started @296799ms
2024-11-19T00:10:17,793 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:10:17,820 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T00:10:17,823 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T00:10:17,824 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T00:10:17,824 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T00:10:17,824 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T00:10:17,824 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2527c84a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/hadoop.log.dir/,AVAILABLE}
2024-11-19T00:10:17,825 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1cfa6b2d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T00:10:17,843 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:17,846 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:17,885 WARN  [Thread-2465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/cluster_62390c19-61af-6e5f-e735-f904e939586e/data/data1/current/BP-660533999-172.17.0.2-1731975017438/current, will proceed with Du for space computation calculation, 
2024-11-19T00:10:17,885 WARN  [Thread-2466 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/cluster_62390c19-61af-6e5f-e735-f904e939586e/data/data2/current/BP-660533999-172.17.0.2-1731975017438/current, will proceed with Du for space computation calculation, 
2024-11-19T00:10:17,907 WARN  [Thread-2444 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:10:17,909 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd8e4fc40d86861f3 with lease ID 0x3cd0738d07c5cd9: Processing first storage report for DS-1b469b29-3636-459e-9646-379e0b922b3d from datanode DatanodeRegistration(127.0.0.1:46137, datanodeUuid=4a6aabdb-1aa5-4c71-9c8c-8e5c6a91e81a, infoPort=46485, infoSecurePort=0, ipcPort=40597, storageInfo=lv=-57;cid=testClusterID;nsid=185879061;c=1731975017438)
2024-11-19T00:10:17,910 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd8e4fc40d86861f3 with lease ID 0x3cd0738d07c5cd9: from storage DS-1b469b29-3636-459e-9646-379e0b922b3d node DatanodeRegistration(127.0.0.1:46137, datanodeUuid=4a6aabdb-1aa5-4c71-9c8c-8e5c6a91e81a, infoPort=46485, infoSecurePort=0, ipcPort=40597, storageInfo=lv=-57;cid=testClusterID;nsid=185879061;c=1731975017438), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:10:17,910 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd8e4fc40d86861f3 with lease ID 0x3cd0738d07c5cd9: Processing first storage report for DS-426de2b0-3bd2-4f89-bfcf-6fe743f174df from datanode DatanodeRegistration(127.0.0.1:46137, datanodeUuid=4a6aabdb-1aa5-4c71-9c8c-8e5c6a91e81a, infoPort=46485, infoSecurePort=0, ipcPort=40597, storageInfo=lv=-57;cid=testClusterID;nsid=185879061;c=1731975017438)
2024-11-19T00:10:17,910 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd8e4fc40d86861f3 with lease ID 0x3cd0738d07c5cd9: from storage DS-426de2b0-3bd2-4f89-bfcf-6fe743f174df node DatanodeRegistration(127.0.0.1:46137, datanodeUuid=4a6aabdb-1aa5-4c71-9c8c-8e5c6a91e81a, infoPort=46485, infoSecurePort=0, ipcPort=40597, storageInfo=lv=-57;cid=testClusterID;nsid=185879061;c=1731975017438), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:10:17,950 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54008d53{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/java.io.tmpdir/jetty-localhost-36885-hadoop-hdfs-3_4_1-tests_jar-_-any-10497738927770146552/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:10:17,950 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4cb9612e{HTTP/1.1, (http/1.1)}{localhost:36885}
2024-11-19T00:10:17,950 INFO  [Time-limited test {}] server.Server(415): Started @296958ms
2024-11-19T00:10:17,952 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T00:10:18,030 WARN  [Thread-2491 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/cluster_62390c19-61af-6e5f-e735-f904e939586e/data/data3/current/BP-660533999-172.17.0.2-1731975017438/current, will proceed with Du for space computation calculation, 
2024-11-19T00:10:18,030 WARN  [Thread-2492 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/cluster_62390c19-61af-6e5f-e735-f904e939586e/data/data4/current/BP-660533999-172.17.0.2-1731975017438/current, will proceed with Du for space computation calculation, 
2024-11-19T00:10:18,047 WARN  [Thread-2480 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T00:10:18,049 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6318141c95164bf8 with lease ID 0x3cd0738d07c5cda: Processing first storage report for DS-75942142-13f1-4f07-a468-502f783e0961 from datanode DatanodeRegistration(127.0.0.1:42589, datanodeUuid=bf03241a-faa4-4527-abe6-877198bf9f7a, infoPort=33873, infoSecurePort=0, ipcPort=44803, storageInfo=lv=-57;cid=testClusterID;nsid=185879061;c=1731975017438)
2024-11-19T00:10:18,049 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6318141c95164bf8 with lease ID 0x3cd0738d07c5cda: from storage DS-75942142-13f1-4f07-a468-502f783e0961 node DatanodeRegistration(127.0.0.1:42589, datanodeUuid=bf03241a-faa4-4527-abe6-877198bf9f7a, infoPort=33873, infoSecurePort=0, ipcPort=44803, storageInfo=lv=-57;cid=testClusterID;nsid=185879061;c=1731975017438), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:10:18,049 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6318141c95164bf8 with lease ID 0x3cd0738d07c5cda: Processing first storage report for DS-8a0dd1c8-a865-4203-bada-4665dba4f271 from datanode DatanodeRegistration(127.0.0.1:42589, datanodeUuid=bf03241a-faa4-4527-abe6-877198bf9f7a, infoPort=33873, infoSecurePort=0, ipcPort=44803, storageInfo=lv=-57;cid=testClusterID;nsid=185879061;c=1731975017438)
2024-11-19T00:10:18,049 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6318141c95164bf8 with lease ID 0x3cd0738d07c5cda: from storage DS-8a0dd1c8-a865-4203-bada-4665dba4f271 node DatanodeRegistration(127.0.0.1:42589, datanodeUuid=bf03241a-faa4-4527-abe6-877198bf9f7a, infoPort=33873, infoSecurePort=0, ipcPort=44803, storageInfo=lv=-57;cid=testClusterID;nsid=185879061;c=1731975017438), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T00:10:18,074 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230
2024-11-19T00:10:18,076 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/cluster_62390c19-61af-6e5f-e735-f904e939586e/zookeeper_0, clientPort=56718, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/cluster_62390c19-61af-6e5f-e735-f904e939586e/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/cluster_62390c19-61af-6e5f-e735-f904e939586e/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-19T00:10:18,077 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56718
2024-11-19T00:10:18,077 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:10:18,079 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:10:18,087 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741825_1001 (size=7)
2024-11-19T00:10:18,087 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741825_1001 (size=7)
2024-11-19T00:10:18,088 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba with version=8
2024-11-19T00:10:18,088 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34343/user/jenkins/test-data/0c3a01d1-abae-da27-429a-9692c7ecffe4/hbase-staging
2024-11-19T00:10:18,090 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/9c732c6543ff:0 server-side Connection retries=45
2024-11-19T00:10:18,090 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:10:18,091 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T00:10:18,091 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T00:10:18,091 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:10:18,091 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T00:10:18,091 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-19T00:10:18,091 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T00:10:18,091 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35239
2024-11-19T00:10:18,092 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35239 connecting to ZooKeeper ensemble=127.0.0.1:56718
2024-11-19T00:10:18,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:352390x0, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T00:10:18,099 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35239-0x10108546b060000 connected
2024-11-19T00:10:18,113 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:10:18,115 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:10:18,117 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:10:18,117 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba, hbase.cluster.distributed=false
2024-11-19T00:10:18,119 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T00:10:18,121 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35239
2024-11-19T00:10:18,121 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35239
2024-11-19T00:10:18,125 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35239
2024-11-19T00:10:18,125 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35239
2024-11-19T00:10:18,125 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35239
2024-11-19T00:10:18,143 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/9c732c6543ff:0 server-side Connection retries=45
2024-11-19T00:10:18,143 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:10:18,143 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T00:10:18,143 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T00:10:18,143 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T00:10:18,143 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T00:10:18,143 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-19T00:10:18,143 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T00:10:18,144 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39129
2024-11-19T00:10:18,145 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39129 connecting to ZooKeeper ensemble=127.0.0.1:56718
2024-11-19T00:10:18,146 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:10:18,147 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:10:18,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:391290x0, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T00:10:18,152 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39129-0x10108546b060001 connected
2024-11-19T00:10:18,152 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:10:18,152 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-19T00:10:18,153 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-19T00:10:18,153 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-19T00:10:18,154 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T00:10:18,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39129
2024-11-19T00:10:18,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39129
2024-11-19T00:10:18,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39129
2024-11-19T00:10:18,158 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39129
2024-11-19T00:10:18,159 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39129
2024-11-19T00:10:18,171 DEBUG [M:0;9c732c6543ff:35239 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9c732c6543ff:35239
2024-11-19T00:10:18,171 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9c732c6543ff,35239,1731975018090
2024-11-19T00:10:18,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:10:18,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:10:18,173 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9c732c6543ff,35239,1731975018090
2024-11-19T00:10:18,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-19T00:10:18,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:18,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:18,176 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-19T00:10:18,176 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9c732c6543ff,35239,1731975018090 from backup master directory
2024-11-19T00:10:18,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:10:18,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9c732c6543ff,35239,1731975018090
2024-11-19T00:10:18,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T00:10:18,178 WARN  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T00:10:18,178 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9c732c6543ff,35239,1731975018090
2024-11-19T00:10:18,181 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/hbase.id] with ID: 834459e7-c594-4f21-b9f6-7eeaddb39992
2024-11-19T00:10:18,181 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/.tmp/hbase.id
2024-11-19T00:10:18,190 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741826_1002 (size=42)
2024-11-19T00:10:18,191 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741826_1002 (size=42)
2024-11-19T00:10:18,191 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/.tmp/hbase.id]:[hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/hbase.id]
2024-11-19T00:10:18,203 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:10:18,203 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-19T00:10:18,204 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-19T00:10:18,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:18,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:18,213 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741827_1003 (size=196)
2024-11-19T00:10:18,213 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741827_1003 (size=196)
2024-11-19T00:10:18,214 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T00:10:18,215 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-19T00:10:18,215 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:10:18,230 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741828_1004 (size=1189)
2024-11-19T00:10:18,230 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741828_1004 (size=1189)
2024-11-19T00:10:18,231 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store
2024-11-19T00:10:18,238 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741829_1005 (size=34)
2024-11-19T00:10:18,239 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741829_1005 (size=34)
2024-11-19T00:10:18,239 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:10:18,239 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T00:10:18,239 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:10:18,239 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:10:18,239 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T00:10:18,239 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:10:18,239 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:10:18,239 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731975018239Disabling compacts and flushes for region at 1731975018239Disabling writes for close at 1731975018239Writing region close event to WAL at 1731975018239Closed at 1731975018239
2024-11-19T00:10:18,240 WARN  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/.initializing
2024-11-19T00:10:18,240 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/WALs/9c732c6543ff,35239,1731975018090
2024-11-19T00:10:18,243 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C35239%2C1731975018090, suffix=, logDir=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/WALs/9c732c6543ff,35239,1731975018090, archiveDir=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/oldWALs, maxLogs=10
2024-11-19T00:10:18,243 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C35239%2C1731975018090.1731975018243
2024-11-19T00:10:18,247 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/WALs/9c732c6543ff,35239,1731975018090/9c732c6543ff%2C35239%2C1731975018090.1731975018243
2024-11-19T00:10:18,248 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33873:33873),(127.0.0.1/127.0.0.1:46485:46485)]
2024-11-19T00:10:18,249 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:10:18,249 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:10:18,249 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:10:18,249 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:10:18,252 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:10:18,254 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-19T00:10:18,254 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:10:18,254 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:10:18,254 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:10:18,255 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-19T00:10:18,255 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:10:18,256 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:10:18,256 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:10:18,257 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-19T00:10:18,257 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:10:18,257 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:10:18,257 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T00:10:18,258 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-19T00:10:18,258 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:10:18,259 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T00:10:18,259 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:10:18,259 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:10:18,260 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:10:18,261 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:10:18,261 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:10:18,261 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-19T00:10:18,262 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T00:10:18,264 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:10:18,264 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=756949, jitterRate=-0.037489935755729675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-19T00:10:18,265 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731975018249Initializing all the Stores at 1731975018250 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975018250Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975018252 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975018252Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975018252Cleaning up temporary data from old regions at 1731975018261 (+9 ms)Region opened successfully at 1731975018265 (+4 ms)
2024-11-19T00:10:18,269 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-19T00:10:18,272 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59dd6fe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0
2024-11-19T00:10:18,273 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-19T00:10:18,273 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-19T00:10:18,273 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-19T00:10:18,273 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-19T00:10:18,273 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-19T00:10:18,274 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-19T00:10:18,274 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-19T00:10:18,276 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-19T00:10:18,277 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-19T00:10:18,278 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-19T00:10:18,279 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-19T00:10:18,279 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-19T00:10:18,280 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-19T00:10:18,281 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-19T00:10:18,282 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-19T00:10:18,283 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-19T00:10:18,284 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-19T00:10:18,286 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-19T00:10:18,289 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-19T00:10:18,290 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-19T00:10:18,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T00:10:18,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T00:10:18,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:18,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:18,293 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9c732c6543ff,35239,1731975018090, sessionid=0x10108546b060000, setting cluster-up flag (Was=false)
2024-11-19T00:10:18,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:18,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:18,301 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-19T00:10:18,303 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,35239,1731975018090
2024-11-19T00:10:18,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:18,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:18,313 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-19T00:10:18,314 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,35239,1731975018090
2024-11-19T00:10:18,316 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-19T00:10:18,317 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-19T00:10:18,318 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-19T00:10:18,318 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-19T00:10:18,318 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9c732c6543ff,35239,1731975018090
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-19T00:10:18,320 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:10:18,320 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:10:18,320 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:10:18,320 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5
2024-11-19T00:10:18,320 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9c732c6543ff:0, corePoolSize=10, maxPoolSize=10
2024-11-19T00:10:18,320 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:10:18,320 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2
2024-11-19T00:10:18,320 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:10:18,323 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731975048323
2024-11-19T00:10:18,323 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-19T00:10:18,323 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-19T00:10:18,323 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:10:18,323 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-19T00:10:18,323 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-19T00:10:18,323 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-19T00:10:18,323 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-19T00:10:18,323 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-19T00:10:18,325 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:10:18,325 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-19T00:10:18,325 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,326 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-19T00:10:18,326 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-19T00:10:18,326 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-19T00:10:18,326 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-19T00:10:18,326 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-19T00:10:18,326 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731975018326,5,FailOnTimeoutGroup]
2024-11-19T00:10:18,329 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731975018326,5,FailOnTimeoutGroup]
2024-11-19T00:10:18,329 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,329 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-19T00:10:18,329 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,329 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,339 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741831_1007 (size=1321)
2024-11-19T00:10:18,339 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741831_1007 (size=1321)
2024-11-19T00:10:18,340 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-19T00:10:18,340 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba
2024-11-19T00:10:18,354 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741832_1008 (size=32)
2024-11-19T00:10:18,355 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741832_1008 (size=32)
2024-11-19T00:10:18,356 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:10:18,357 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T00:10:18,358 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T00:10:18,358 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:10:18,359 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:10:18,359 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T00:10:18,360 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T00:10:18,360 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:10:18,360 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:10:18,360 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T00:10:18,361 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.HRegionServer(746): ClusterId : 834459e7-c594-4f21-b9f6-7eeaddb39992
2024-11-19T00:10:18,361 DEBUG [RS:0;9c732c6543ff:39129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-19T00:10:18,361 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T00:10:18,361 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:10:18,362 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:10:18,362 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T00:10:18,363 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T00:10:18,363 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:10:18,363 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:10:18,363 DEBUG [RS:0;9c732c6543ff:39129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-19T00:10:18,363 DEBUG [RS:0;9c732c6543ff:39129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-19T00:10:18,363 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T00:10:18,364 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/data/hbase/meta/1588230740
2024-11-19T00:10:18,364 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/data/hbase/meta/1588230740
2024-11-19T00:10:18,365 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T00:10:18,365 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T00:10:18,366 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T00:10:18,366 DEBUG [RS:0;9c732c6543ff:39129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-19T00:10:18,366 DEBUG [RS:0;9c732c6543ff:39129 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d93282, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0
2024-11-19T00:10:18,367 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T00:10:18,370 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T00:10:18,370 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=776458, jitterRate=-0.01268342137336731}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T00:10:18,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731975018356Initializing all the Stores at 1731975018357 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975018357Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975018357Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975018357Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975018357Cleaning up temporary data from old regions at 1731975018365 (+8 ms)Region opened successfully at 1731975018371 (+6 ms)
2024-11-19T00:10:18,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T00:10:18,371 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T00:10:18,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T00:10:18,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T00:10:18,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T00:10:18,371 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T00:10:18,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731975018371Disabling compacts and flushes for region at 1731975018371Disabling writes for close at 1731975018371Writing region close event to WAL at 1731975018371Closed at 1731975018371
2024-11-19T00:10:18,373 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:10:18,373 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-19T00:10:18,373 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-19T00:10:18,374 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T00:10:18,375 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-19T00:10:18,385 DEBUG [RS:0;9c732c6543ff:39129 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9c732c6543ff:39129
2024-11-19T00:10:18,385 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-19T00:10:18,385 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-19T00:10:18,385 DEBUG [RS:0;9c732c6543ff:39129 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-19T00:10:18,386 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.HRegionServer(2659): reportForDuty to master=9c732c6543ff,35239,1731975018090 with port=39129, startcode=1731975018143
2024-11-19T00:10:18,386 DEBUG [RS:0;9c732c6543ff:39129 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-19T00:10:18,388 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55791, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-11-19T00:10:18,389 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35239 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9c732c6543ff,39129,1731975018143
2024-11-19T00:10:18,389 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35239 {}] master.ServerManager(517): Registering regionserver=9c732c6543ff,39129,1731975018143
2024-11-19T00:10:18,390 DEBUG [RS:0;9c732c6543ff:39129 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba
2024-11-19T00:10:18,390 DEBUG [RS:0;9c732c6543ff:39129 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41107
2024-11-19T00:10:18,390 DEBUG [RS:0;9c732c6543ff:39129 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-19T00:10:18,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T00:10:18,393 DEBUG [RS:0;9c732c6543ff:39129 {}] zookeeper.ZKUtil(111): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9c732c6543ff,39129,1731975018143
2024-11-19T00:10:18,393 WARN  [RS:0;9c732c6543ff:39129 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T00:10:18,393 INFO  [RS:0;9c732c6543ff:39129 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:10:18,393 DEBUG [RS:0;9c732c6543ff:39129 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/WALs/9c732c6543ff,39129,1731975018143
2024-11-19T00:10:18,393 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9c732c6543ff,39129,1731975018143]
2024-11-19T00:10:18,396 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-19T00:10:18,399 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-19T00:10:18,399 INFO  [RS:0;9c732c6543ff:39129 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T00:10:18,399 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,399 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-19T00:10:18,400 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-19T00:10:18,400 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,400 DEBUG [RS:0;9c732c6543ff:39129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:10:18,400 DEBUG [RS:0;9c732c6543ff:39129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:10:18,400 DEBUG [RS:0;9c732c6543ff:39129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:10:18,400 DEBUG [RS:0;9c732c6543ff:39129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:10:18,400 DEBUG [RS:0;9c732c6543ff:39129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:10:18,400 DEBUG [RS:0;9c732c6543ff:39129 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2
2024-11-19T00:10:18,400 DEBUG [RS:0;9c732c6543ff:39129 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:10:18,400 DEBUG [RS:0;9c732c6543ff:39129 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:10:18,400 DEBUG [RS:0;9c732c6543ff:39129 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:10:18,400 DEBUG [RS:0;9c732c6543ff:39129 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:10:18,400 DEBUG [RS:0;9c732c6543ff:39129 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:10:18,400 DEBUG [RS:0;9c732c6543ff:39129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1
2024-11-19T00:10:18,400 DEBUG [RS:0;9c732c6543ff:39129 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3
2024-11-19T00:10:18,400 DEBUG [RS:0;9c732c6543ff:39129 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3
2024-11-19T00:10:18,401 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,401 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,401 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,401 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,401 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,401 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,39129,1731975018143-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T00:10:18,416 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-19T00:10:18,416 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,39129,1731975018143-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,416 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,416 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.Replication(171): 9c732c6543ff,39129,1731975018143 started
2024-11-19T00:10:18,430 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,430 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.HRegionServer(1482): Serving as 9c732c6543ff,39129,1731975018143, RpcServer on 9c732c6543ff/172.17.0.2:39129, sessionid=0x10108546b060001
2024-11-19T00:10:18,431 DEBUG [RS:0;9c732c6543ff:39129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-19T00:10:18,431 DEBUG [RS:0;9c732c6543ff:39129 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9c732c6543ff,39129,1731975018143
2024-11-19T00:10:18,431 DEBUG [RS:0;9c732c6543ff:39129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,39129,1731975018143'
2024-11-19T00:10:18,431 DEBUG [RS:0;9c732c6543ff:39129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-19T00:10:18,431 DEBUG [RS:0;9c732c6543ff:39129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-19T00:10:18,432 DEBUG [RS:0;9c732c6543ff:39129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-19T00:10:18,432 DEBUG [RS:0;9c732c6543ff:39129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-19T00:10:18,432 DEBUG [RS:0;9c732c6543ff:39129 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9c732c6543ff,39129,1731975018143
2024-11-19T00:10:18,432 DEBUG [RS:0;9c732c6543ff:39129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,39129,1731975018143'
2024-11-19T00:10:18,432 DEBUG [RS:0;9c732c6543ff:39129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-19T00:10:18,432 DEBUG [RS:0;9c732c6543ff:39129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-19T00:10:18,432 DEBUG [RS:0;9c732c6543ff:39129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-19T00:10:18,432 INFO  [RS:0;9c732c6543ff:39129 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-19T00:10:18,432 INFO  [RS:0;9c732c6543ff:39129 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-19T00:10:18,525 WARN  [9c732c6543ff:35239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-19T00:10:18,534 INFO  [RS:0;9c732c6543ff:39129 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C39129%2C1731975018143, suffix=, logDir=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/WALs/9c732c6543ff,39129,1731975018143, archiveDir=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/oldWALs, maxLogs=32
2024-11-19T00:10:18,535 INFO  [RS:0;9c732c6543ff:39129 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C39129%2C1731975018143.1731975018535
2024-11-19T00:10:18,540 INFO  [RS:0;9c732c6543ff:39129 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/WALs/9c732c6543ff,39129,1731975018143/9c732c6543ff%2C39129%2C1731975018143.1731975018535
2024-11-19T00:10:18,541 DEBUG [RS:0;9c732c6543ff:39129 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46485:46485),(127.0.0.1/127.0.0.1:33873:33873)]
2024-11-19T00:10:18,775 DEBUG [9c732c6543ff:35239 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-19T00:10:18,776 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9c732c6543ff,39129,1731975018143
2024-11-19T00:10:18,777 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,39129,1731975018143, state=OPENING
2024-11-19T00:10:18,779 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-19T00:10:18,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:18,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:18,781 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:10:18,781 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:10:18,781 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T00:10:18,781 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,39129,1731975018143}]
2024-11-19T00:10:18,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.1731974820245
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:18,844 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42069,1731974820802/9c732c6543ff%2C42069%2C1731974820802.1731974821088
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:18,847 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43835/user/jenkins/test-data/bcea56e0-869e-3ab6-9b82-8a6f18cf4c35/WALs/9c732c6543ff,42145,1731974819848/9c732c6543ff%2C42145%2C1731974819848.meta.1731974820639.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T00:10:18,934 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-19T00:10:18,935 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51275, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-19T00:10:18,939 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-19T00:10:18,939 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:10:18,940 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C39129%2C1731975018143.meta, suffix=.meta, logDir=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/WALs/9c732c6543ff,39129,1731975018143, archiveDir=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/oldWALs, maxLogs=32
2024-11-19T00:10:18,941 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 9c732c6543ff%2C39129%2C1731975018143.meta.1731975018941.meta
2024-11-19T00:10:18,946 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/WALs/9c732c6543ff,39129,1731975018143/9c732c6543ff%2C39129%2C1731975018143.meta.1731975018941.meta
2024-11-19T00:10:18,953 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46485:46485),(127.0.0.1/127.0.0.1:33873:33873)]
2024-11-19T00:10:18,957 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-19T00:10:18,958 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-19T00:10:18,958 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-19T00:10:18,958 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-19T00:10:18,958 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-19T00:10:18,958 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T00:10:18,958 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-19T00:10:18,958 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-19T00:10:18,959 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T00:10:18,960 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T00:10:18,960 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:10:18,960 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:10:18,960 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T00:10:18,961 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T00:10:18,961 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:10:18,962 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:10:18,962 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T00:10:18,962 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T00:10:18,962 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:10:18,963 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:10:18,963 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T00:10:18,963 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T00:10:18,963 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T00:10:18,964 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T00:10:18,964 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T00:10:18,964 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/data/hbase/meta/1588230740
2024-11-19T00:10:18,965 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/data/hbase/meta/1588230740
2024-11-19T00:10:18,966 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T00:10:18,966 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T00:10:18,966 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T00:10:18,968 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T00:10:18,968 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=864334, jitterRate=0.09905776381492615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T00:10:18,968 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-19T00:10:18,969 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731975018958Writing region info on filesystem at 1731975018958Initializing all the Stores at 1731975018959 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975018959Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975018959Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975018959Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975018959Cleaning up temporary data from old regions at 1731975018966 (+7 ms)Running coprocessor post-open hooks at 1731975018968 (+2 ms)Region opened successfully at 1731975018969 (+1 ms)
2024-11-19T00:10:18,970 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731975018933
2024-11-19T00:10:18,972 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-19T00:10:18,972 INFO  [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-19T00:10:18,972 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,39129,1731975018143
2024-11-19T00:10:18,973 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,39129,1731975018143, state=OPEN
2024-11-19T00:10:18,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T00:10:18,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T00:10:18,977 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9c732c6543ff,39129,1731975018143
2024-11-19T00:10:18,977 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:10:18,977 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T00:10:18,980 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-19T00:10:18,980 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,39129,1731975018143 in 196 msec
2024-11-19T00:10:18,982 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-19T00:10:18,982 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 607 msec
2024-11-19T00:10:18,982 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T00:10:18,982 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-19T00:10:18,984 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T00:10:18,984 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,39129,1731975018143, seqNum=-1]
2024-11-19T00:10:18,984 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T00:10:18,985 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52973, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T00:10:18,989 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 672 msec
2024-11-19T00:10:18,989 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731975018989, completionTime=-1
2024-11-19T00:10:18,989 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-19T00:10:18,989 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-19T00:10:18,991 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-19T00:10:18,991 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731975078991
2024-11-19T00:10:18,991 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731975138991
2024-11-19T00:10:18,991 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-11-19T00:10:18,991 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,35239,1731975018090-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,991 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,35239,1731975018090-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,991 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,35239,1731975018090-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,991 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9c732c6543ff:35239, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,992 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,992 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:18,993 DEBUG [master/9c732c6543ff:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-19T00:10:18,995 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.817sec
2024-11-19T00:10:18,995 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-19T00:10:18,995 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-19T00:10:18,995 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-19T00:10:18,995 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-19T00:10:18,995 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-19T00:10:18,995 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,35239,1731975018090-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T00:10:18,995 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,35239,1731975018090-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-19T00:10:18,997 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-19T00:10:18,997 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-19T00:10:18,997 INFO  [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,35239,1731975018090-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T00:10:19,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43bb2f4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:10:19,061 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9c732c6543ff,35239,-1 for getting cluster id
2024-11-19T00:10:19,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-19T00:10:19,062 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '834459e7-c594-4f21-b9f6-7eeaddb39992'
2024-11-19T00:10:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-19T00:10:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "834459e7-c594-4f21-b9f6-7eeaddb39992"

2024-11-19T00:10:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10fa033b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:10:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9c732c6543ff,35239,-1]
2024-11-19T00:10:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-19T00:10:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:10:19,064 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-19T00:10:19,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78fae436, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T00:10:19,065 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T00:10:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,39129,1731975018143, seqNum=-1]
2024-11-19T00:10:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T00:10:19,067 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T00:10:19,069 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9c732c6543ff,35239,1731975018090
2024-11-19T00:10:19,069 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T00:10:19,071 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-19T00:10:19,071 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T00:10:19,073 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/WALs/test.com,8080,1, archiveDir=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/oldWALs, maxLogs=32
2024-11-19T00:10:19,074 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731975019074
2024-11-19T00:10:19,078 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/WALs/test.com,8080,1/test.com%2C8080%2C1.1731975019074
2024-11-19T00:10:19,079 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46485:46485),(127.0.0.1/127.0.0.1:33873:33873)]
2024-11-19T00:10:19,080 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731975019080
2024-11-19T00:10:19,084 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,084 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,084 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,084 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,085 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,085 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/WALs/test.com,8080,1/test.com%2C8080%2C1.1731975019074 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/WALs/test.com,8080,1/test.com%2C8080%2C1.1731975019080
2024-11-19T00:10:19,086 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46485:46485),(127.0.0.1/127.0.0.1:33873:33873)]
2024-11-19T00:10:19,086 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/WALs/test.com,8080,1/test.com%2C8080%2C1.1731975019074 is not closed yet, will try archiving it next time
2024-11-19T00:10:19,086 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,086 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741835_1011 (size=93)
2024-11-19T00:10:19,087 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741835_1011 (size=93)
2024-11-19T00:10:19,087 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/WALs/test.com,8080,1/test.com%2C8080%2C1.1731975019074 to hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/oldWALs/test.com%2C8080%2C1.1731975019074
2024-11-19T00:10:19,088 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,088 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,088 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,088 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,090 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741836_1012 (size=93)
2024-11-19T00:10:19,090 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741836_1012 (size=93)
2024-11-19T00:10:19,092 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/oldWALs
2024-11-19T00:10:19,092 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1731975019080)
2024-11-19T00:10:19,092 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-19T00:10:19,092 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T00:10:19,092 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:10:19,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:10:19,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:10:19,092 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-19T00:10:19,092 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-19T00:10:19,092 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1925418755, stopped=false
2024-11-19T00:10:19,092 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9c732c6543ff,35239,1731975018090
2024-11-19T00:10:19,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T00:10:19,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T00:10:19,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:19,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:19,094 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T00:10:19,094 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T00:10:19,094 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:10:19,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:10:19,095 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9c732c6543ff,39129,1731975018143' *****
2024-11-19T00:10:19,095 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-19T00:10:19,095 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:10:19,095 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-19T00:10:19,095 INFO  [RS:0;9c732c6543ff:39129 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-19T00:10:19,095 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-19T00:10:19,095 INFO  [RS:0;9c732c6543ff:39129 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-19T00:10:19,095 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.HRegionServer(959): stopping server 9c732c6543ff,39129,1731975018143
2024-11-19T00:10:19,095 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T00:10:19,095 INFO  [RS:0;9c732c6543ff:39129 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9c732c6543ff:39129.
2024-11-19T00:10:19,095 DEBUG [RS:0;9c732c6543ff:39129 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T00:10:19,095 DEBUG [RS:0;9c732c6543ff:39129 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:10:19,095 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-19T00:10:19,095 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-19T00:10:19,095 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-19T00:10:19,095 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-19T00:10:19,096 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-19T00:10:19,096 DEBUG [RS:0;9c732c6543ff:39129 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-19T00:10:19,096 DEBUG [RS:0;9c732c6543ff:39129 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-19T00:10:19,096 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T00:10:19,096 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T00:10:19,096 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T00:10:19,096 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T00:10:19,096 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T00:10:19,096 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T00:10:19,096 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-19T00:10:19,112 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/data/hbase/meta/1588230740/.tmp/ns/cf8808d537984a019700060fe0244c80 is 43, key is default/ns:d/1731975018985/Put/seqid=0
2024-11-19T00:10:19,116 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741837_1013 (size=5153)
2024-11-19T00:10:19,116 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741837_1013 (size=5153)
2024-11-19T00:10:19,116 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/data/hbase/meta/1588230740/.tmp/ns/cf8808d537984a019700060fe0244c80
2024-11-19T00:10:19,121 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/data/hbase/meta/1588230740/.tmp/ns/cf8808d537984a019700060fe0244c80 as hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/data/hbase/meta/1588230740/ns/cf8808d537984a019700060fe0244c80
2024-11-19T00:10:19,125 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/data/hbase/meta/1588230740/ns/cf8808d537984a019700060fe0244c80, entries=2, sequenceid=6, filesize=5.0 K
2024-11-19T00:10:19,126 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=6, compaction requested=false
2024-11-19T00:10:19,129 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-19T00:10:19,130 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T00:10:19,130 INFO  [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T00:10:19,130 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731975019096Running coprocessor pre-close hooks at 1731975019096Disabling compacts and flushes for region at 1731975019096Disabling writes for close at 1731975019096Obtaining lock to block concurrent updates at 1731975019096Preparing flush snapshotting stores in 1588230740 at 1731975019096Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731975019096Flushing stores of hbase:meta,,1.1588230740 at 1731975019097 (+1 ms)Flushing 1588230740/ns: creating writer at 1731975019097Flushing 1588230740/ns: appending metadata at 1731975019111 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731975019111Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60b98acb: reopening flushed file at 1731975019121 (+10 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=6, compaction requested=false at 1731975019126 (+5 ms)Writing region close event to WAL at 1731975019127 (+1 ms)Running coprocessor post-close hooks at 1731975019130 (+3 ms)Closed at 1731975019130
2024-11-19T00:10:19,130 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-19T00:10:19,296 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.HRegionServer(976): stopping server 9c732c6543ff,39129,1731975018143; all regions closed.
2024-11-19T00:10:19,297 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,297 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,297 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,297 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,297 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,299 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741834_1010 (size=1152)
2024-11-19T00:10:19,299 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741834_1010 (size=1152)
2024-11-19T00:10:19,301 DEBUG [RS:0;9c732c6543ff:39129 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/oldWALs
2024-11-19T00:10:19,301 INFO  [RS:0;9c732c6543ff:39129 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9c732c6543ff%2C39129%2C1731975018143.meta:.meta(num 1731975018941)
2024-11-19T00:10:19,302 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,302 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,302 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,302 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,302 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,303 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741833_1009 (size=93)
2024-11-19T00:10:19,304 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741833_1009 (size=93)
2024-11-19T00:10:19,306 DEBUG [RS:0;9c732c6543ff:39129 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/oldWALs
2024-11-19T00:10:19,306 INFO  [RS:0;9c732c6543ff:39129 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9c732c6543ff%2C39129%2C1731975018143:(num 1731975018535)
2024-11-19T00:10:19,306 DEBUG [RS:0;9c732c6543ff:39129 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T00:10:19,306 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T00:10:19,306 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T00:10:19,306 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.ChoreService(370): Chore service for: regionserver/9c732c6543ff:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-19T00:10:19,306 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T00:10:19,306 INFO  [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T00:10:19,306 INFO  [RS:0;9c732c6543ff:39129 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39129
2024-11-19T00:10:19,308 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T00:10:19,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9c732c6543ff,39129,1731975018143
2024-11-19T00:10:19,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T00:10:19,310 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9c732c6543ff,39129,1731975018143]
2024-11-19T00:10:19,311 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9c732c6543ff,39129,1731975018143 already deleted, retry=false
2024-11-19T00:10:19,311 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9c732c6543ff,39129,1731975018143 expired; onlineServers=0
2024-11-19T00:10:19,311 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9c732c6543ff,35239,1731975018090' *****
2024-11-19T00:10:19,311 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-19T00:10:19,311 INFO  [M:0;9c732c6543ff:35239 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T00:10:19,311 INFO  [M:0;9c732c6543ff:35239 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T00:10:19,311 DEBUG [M:0;9c732c6543ff:35239 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-19T00:10:19,311 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-19T00:10:19,311 DEBUG [M:0;9c732c6543ff:35239 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-19T00:10:19,311 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731975018326 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731975018326,5,FailOnTimeoutGroup]
2024-11-19T00:10:19,311 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731975018326 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731975018326,5,FailOnTimeoutGroup]
2024-11-19T00:10:19,311 INFO  [M:0;9c732c6543ff:35239 {}] hbase.ChoreService(370): Chore service for: master/9c732c6543ff:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-19T00:10:19,312 INFO  [M:0;9c732c6543ff:35239 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T00:10:19,312 DEBUG [M:0;9c732c6543ff:35239 {}] master.HMaster(1795): Stopping service threads
2024-11-19T00:10:19,312 INFO  [M:0;9c732c6543ff:35239 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-19T00:10:19,312 INFO  [M:0;9c732c6543ff:35239 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T00:10:19,312 INFO  [M:0;9c732c6543ff:35239 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-19T00:10:19,312 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-19T00:10:19,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-19T00:10:19,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T00:10:19,313 DEBUG [M:0;9c732c6543ff:35239 {}] zookeeper.ZKUtil(347): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-19T00:10:19,313 WARN  [M:0;9c732c6543ff:35239 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-19T00:10:19,313 INFO  [M:0;9c732c6543ff:35239 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/.lastflushedseqids
2024-11-19T00:10:19,318 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741838_1014 (size=99)
2024-11-19T00:10:19,318 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741838_1014 (size=99)
2024-11-19T00:10:19,321 INFO  [M:0;9c732c6543ff:35239 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-19T00:10:19,321 INFO  [M:0;9c732c6543ff:35239 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-19T00:10:19,322 DEBUG [M:0;9c732c6543ff:35239 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T00:10:19,322 INFO  [M:0;9c732c6543ff:35239 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:10:19,322 DEBUG [M:0;9c732c6543ff:35239 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:10:19,322 DEBUG [M:0;9c732c6543ff:35239 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T00:10:19,322 DEBUG [M:0;9c732c6543ff:35239 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:10:19,322 INFO  [M:0;9c732c6543ff:35239 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-19T00:10:19,337 DEBUG [M:0;9c732c6543ff:35239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/74920b5309374d17837e547380acd952 is 82, key is hbase:meta,,1/info:regioninfo/1731975018972/Put/seqid=0
2024-11-19T00:10:19,342 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741839_1015 (size=5672)
2024-11-19T00:10:19,342 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741839_1015 (size=5672)
2024-11-19T00:10:19,342 INFO  [M:0;9c732c6543ff:35239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/74920b5309374d17837e547380acd952
2024-11-19T00:10:19,368 DEBUG [M:0;9c732c6543ff:35239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e973b707f0a04c9b8a8caf626d1962f3 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731975018989/Put/seqid=0
2024-11-19T00:10:19,372 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741840_1016 (size=5275)
2024-11-19T00:10:19,373 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741840_1016 (size=5275)
2024-11-19T00:10:19,373 INFO  [M:0;9c732c6543ff:35239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e973b707f0a04c9b8a8caf626d1962f3
2024-11-19T00:10:19,392 DEBUG [M:0;9c732c6543ff:35239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/517756b911c641399984e6f0a731bde3 is 69, key is 9c732c6543ff,39129,1731975018143/rs:state/1731975018389/Put/seqid=0
2024-11-19T00:10:19,396 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741841_1017 (size=5156)
2024-11-19T00:10:19,397 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741841_1017 (size=5156)
2024-11-19T00:10:19,397 INFO  [M:0;9c732c6543ff:35239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/517756b911c641399984e6f0a731bde3
2024-11-19T00:10:19,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:10:19,410 INFO  [RS:0;9c732c6543ff:39129 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T00:10:19,410 INFO  [RS:0;9c732c6543ff:39129 {}] regionserver.HRegionServer(1031): Exiting; stopping=9c732c6543ff,39129,1731975018143; zookeeper connection closed.
2024-11-19T00:10:19,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39129-0x10108546b060001, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:10:19,410 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4e5f1c2d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4e5f1c2d
2024-11-19T00:10:19,411 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-19T00:10:19,415 DEBUG [M:0;9c732c6543ff:35239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6607501efa694a0ba3141d6854b40a5c is 52, key is load_balancer_on/state:d/1731975019070/Put/seqid=0
2024-11-19T00:10:19,419 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741842_1018 (size=5056)
2024-11-19T00:10:19,420 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741842_1018 (size=5056)
2024-11-19T00:10:19,420 INFO  [M:0;9c732c6543ff:35239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6607501efa694a0ba3141d6854b40a5c
2024-11-19T00:10:19,424 DEBUG [M:0;9c732c6543ff:35239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/74920b5309374d17837e547380acd952 as hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/74920b5309374d17837e547380acd952
2024-11-19T00:10:19,428 INFO  [M:0;9c732c6543ff:35239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/74920b5309374d17837e547380acd952, entries=8, sequenceid=29, filesize=5.5 K
2024-11-19T00:10:19,429 DEBUG [M:0;9c732c6543ff:35239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e973b707f0a04c9b8a8caf626d1962f3 as hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e973b707f0a04c9b8a8caf626d1962f3
2024-11-19T00:10:19,432 INFO  [M:0;9c732c6543ff:35239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e973b707f0a04c9b8a8caf626d1962f3, entries=3, sequenceid=29, filesize=5.2 K
2024-11-19T00:10:19,433 DEBUG [M:0;9c732c6543ff:35239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/517756b911c641399984e6f0a731bde3 as hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/517756b911c641399984e6f0a731bde3
2024-11-19T00:10:19,436 INFO  [M:0;9c732c6543ff:35239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/517756b911c641399984e6f0a731bde3, entries=1, sequenceid=29, filesize=5.0 K
2024-11-19T00:10:19,437 DEBUG [M:0;9c732c6543ff:35239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6607501efa694a0ba3141d6854b40a5c as hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6607501efa694a0ba3141d6854b40a5c
2024-11-19T00:10:19,441 INFO  [M:0;9c732c6543ff:35239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41107/user/jenkins/test-data/b172a3ba-7dea-5a88-4884-5c9eb67505ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6607501efa694a0ba3141d6854b40a5c, entries=1, sequenceid=29, filesize=4.9 K
2024-11-19T00:10:19,442 INFO  [M:0;9c732c6543ff:35239 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 119ms, sequenceid=29, compaction requested=false
2024-11-19T00:10:19,443 INFO  [M:0;9c732c6543ff:35239 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T00:10:19,443 DEBUG [M:0;9c732c6543ff:35239 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731975019322Disabling compacts and flushes for region at 1731975019322Disabling writes for close at 1731975019322Obtaining lock to block concurrent updates at 1731975019322Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731975019322Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731975019322Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731975019323 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731975019323Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731975019337 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731975019337Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731975019347 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731975019367 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731975019367Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731975019377 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731975019391 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731975019391Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731975019401 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731975019415 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731975019415Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ede2697: reopening flushed file at 1731975019424 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38fb7596: reopening flushed file at 1731975019428 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a324ab: reopening flushed file at 1731975019432 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4953a3a8: reopening flushed file at 1731975019437 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 119ms, sequenceid=29, compaction requested=false at 1731975019442 (+5 ms)Writing region close event to WAL at 1731975019443 (+1 ms)Closed at 1731975019443
2024-11-19T00:10:19,443 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,443 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,444 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,444 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,444 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T00:10:19,446 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46137 is added to blk_1073741830_1006 (size=10311)
2024-11-19T00:10:19,446 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42589 is added to blk_1073741830_1006 (size=10311)
2024-11-19T00:10:19,446 INFO  [M:0;9c732c6543ff:35239 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-19T00:10:19,446 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T00:10:19,446 INFO  [M:0;9c732c6543ff:35239 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35239
2024-11-19T00:10:19,447 INFO  [M:0;9c732c6543ff:35239 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T00:10:19,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:10:19,549 INFO  [M:0;9c732c6543ff:35239 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T00:10:19,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35239-0x10108546b060000, quorum=127.0.0.1:56718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T00:10:19,551 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54008d53{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:10:19,551 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4cb9612e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:10:19,551 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:10:19,551 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1cfa6b2d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:10:19,552 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2527c84a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/hadoop.log.dir/,STOPPED}
2024-11-19T00:10:19,553 WARN  [BP-660533999-172.17.0.2-1731975017438 heartbeating to localhost/127.0.0.1:41107 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:10:19,553 WARN  [BP-660533999-172.17.0.2-1731975017438 heartbeating to localhost/127.0.0.1:41107 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-660533999-172.17.0.2-1731975017438 (Datanode Uuid bf03241a-faa4-4527-abe6-877198bf9f7a) service to localhost/127.0.0.1:41107
2024-11-19T00:10:19,553 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:10:19,553 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:10:19,553 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/cluster_62390c19-61af-6e5f-e735-f904e939586e/data/data3/current/BP-660533999-172.17.0.2-1731975017438 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:10:19,553 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/cluster_62390c19-61af-6e5f-e735-f904e939586e/data/data4/current/BP-660533999-172.17.0.2-1731975017438 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:10:19,553 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:10:19,555 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7e09407e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T00:10:19,555 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@61266769{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:10:19,556 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:10:19,556 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@259cffcc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:10:19,556 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7d6e4d22{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/hadoop.log.dir/,STOPPED}
2024-11-19T00:10:19,557 WARN  [BP-660533999-172.17.0.2-1731975017438 heartbeating to localhost/127.0.0.1:41107 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T00:10:19,557 WARN  [BP-660533999-172.17.0.2-1731975017438 heartbeating to localhost/127.0.0.1:41107 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-660533999-172.17.0.2-1731975017438 (Datanode Uuid 4a6aabdb-1aa5-4c71-9c8c-8e5c6a91e81a) service to localhost/127.0.0.1:41107
2024-11-19T00:10:19,557 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T00:10:19,557 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T00:10:19,557 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/cluster_62390c19-61af-6e5f-e735-f904e939586e/data/data1/current/BP-660533999-172.17.0.2-1731975017438 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:10:19,558 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/cluster_62390c19-61af-6e5f-e735-f904e939586e/data/data2/current/BP-660533999-172.17.0.2-1731975017438 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T00:10:19,558 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T00:10:19,564 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6486a7e6{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T00:10:19,564 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34852128{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T00:10:19,564 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T00:10:19,564 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5fab3ee3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T00:10:19,564 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6ce66601{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05df7a32-a760-2ba2-31df-8317717ac230/hadoop.log.dir/,STOPPED}
2024-11-19T00:10:19,570 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-19T00:10:19,583 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-19T00:10:19,592 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=268 (was 230)
Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41107 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41107
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41107
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:41107
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: globalEventExecutor-1-22
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	app//io.netty.util.concurrent.GlobalEventExecutor.takeTask(GlobalEventExecutor.java:113)
	app//io.netty.util.concurrent.GlobalEventExecutor$TaskRunner.run(GlobalEventExecutor.java:259)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:41107
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41107 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41107 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41107
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=535 (was 506) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=245 (was 245), ProcessCount=11 (was 11), AvailableMemoryMB=3528 (was 3545)